From d999428cd51aa239421be4e5e2e22f1357d0ba74 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Tue, 16 Jun 2026 18:22:52 +0300 Subject: [PATCH 1/4] feat: add AsyncPageable, Page, and the Pageable factory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds core pagination support to Dexpace.Sdk.Core: - `Page` — immutable carrier of Values, Status, and Headers for one HTTP page. - `AsyncPageable` — abstract base implementing IAsyncEnumerable with an AsPages() view for per-page metadata. - `Pageable.Create` — factory that builds a lazy pipeline-driven pageable; each page is fetched on-demand through the full HttpPipeline (retry, auth, telemetry apply per page). nextRequest drives iteration; maxPages caps it. Responses are disposed inside a try/finally before yielding each Page so headers are safely captured first. 19 new tests in Dexpace.Sdk.Core.Tests/Pagination/ covering: item flattening, AsPages count/values/metadata, laziness (single send after first-page consume, second send on advance), maxPages cap, null-nextRequest termination, response body disposal, Page constructor guards, and Pageable.Create null-arg guards. Co-Authored-By: Claude Opus 4.8 --- .../Pagination/AsyncPageable.cs | 36 ++ src/Dexpace.Sdk.Core/Pagination/Page.cs | 43 ++ src/Dexpace.Sdk.Core/Pagination/Pageable.cs | 150 ++++++ .../Pagination/PageableTests.cs | 500 ++++++++++++++++++ 4 files changed, 729 insertions(+) create mode 100644 src/Dexpace.Sdk.Core/Pagination/AsyncPageable.cs create mode 100644 src/Dexpace.Sdk.Core/Pagination/Page.cs create mode 100644 src/Dexpace.Sdk.Core/Pagination/Pageable.cs create mode 100644 tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs diff --git a/src/Dexpace.Sdk.Core/Pagination/AsyncPageable.cs b/src/Dexpace.Sdk.Core/Pagination/AsyncPageable.cs new file mode 100644 index 0000000..2676a65 --- /dev/null +++ b/src/Dexpace.Sdk.Core/Pagination/AsyncPageable.cs @@ -0,0 +1,36 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +namespace Dexpace.Sdk.Core.Pagination; + +/// +/// An async-enumerable sequence of items that is backed by a series of HTTP pages. +/// +/// The item type. +/// +/// +/// Consumers can iterate items directly (await foreach (var item in pageable)) or iterate +/// pages via for access to per-page metadata such as status and headers. +/// +/// +/// Enumeration is lazy: the next page is fetched only when the consumer advances past the last +/// item of the current page. Each page send is an independent pipeline invocation. +/// +/// +public abstract class AsyncPageable : IAsyncEnumerable +{ + /// Returns an async sequence of instances. + /// + /// An optional hint for the number of items per page. How (or whether) this is used depends + /// on the concrete implementation. + /// + /// An async sequence of pages. + public abstract IAsyncEnumerable> AsPages(int? pageSizeHint = null); + + /// + /// Returns an enumerator that iterates items from all pages in sequence. + /// + /// A token to cancel enumeration. + /// An over all items across all pages. + public abstract IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default); +} diff --git a/src/Dexpace.Sdk.Core/Pagination/Page.cs b/src/Dexpace.Sdk.Core/Pagination/Page.cs new file mode 100644 index 0000000..28bbd63 --- /dev/null +++ b/src/Dexpace.Sdk.Core/Pagination/Page.cs @@ -0,0 +1,43 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +using Dexpace.Sdk.Core.Http.Common; +using Dexpace.Sdk.Core.Http.Response; + +namespace Dexpace.Sdk.Core.Pagination; + +/// +/// A single page of results returned by a paginated operation. +/// +/// The item type. +/// +/// +/// contains the deserialized items for this page. +/// and are the HTTP metadata captured from the +/// response before it was disposed; they are immutable and safe to retain after iteration. +/// +/// +public sealed class Page +{ + /// Creates a page. + /// The items on this page. + /// The HTTP status of the response that produced this page. + /// The HTTP response headers for this page. + public Page(IReadOnlyList values, Status status, Headers headers) + { + ArgumentNullException.ThrowIfNull(values); + ArgumentNullException.ThrowIfNull(headers); + Values = values; + Status = status; + Headers = headers; + } + + /// The items on this page. + public IReadOnlyList Values { get; } + + /// The HTTP status code of the response that produced this page. + public Status Status { get; } + + /// The HTTP response headers for this page. + public Headers Headers { get; } +} diff --git a/src/Dexpace.Sdk.Core/Pagination/Pageable.cs b/src/Dexpace.Sdk.Core/Pagination/Pageable.cs new file mode 100644 index 0000000..346471b --- /dev/null +++ b/src/Dexpace.Sdk.Core/Pagination/Pageable.cs @@ -0,0 +1,150 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +using System.Runtime.CompilerServices; +using Dexpace.Sdk.Core.Configuration; +using Dexpace.Sdk.Core.Errors; +using Dexpace.Sdk.Core.Http.Common; +using Dexpace.Sdk.Core.Http.Request; +using Dexpace.Sdk.Core.Http.Response; +using Dexpace.Sdk.Core.Pipeline; +using Dexpace.Sdk.Core.Serialization; + +namespace Dexpace.Sdk.Core.Pagination; + +/// +/// Factory methods for creating instances. +/// +public static class Pageable +{ + /// + /// Creates an that fetches pages through + /// , starting with . + /// + /// The deserialized page-envelope type. + /// The item type extracted from each page. + /// The pipeline used for each page request. + /// The initial request to send. + /// The serde used to deserialize each . + /// Client options forwarded to each pipeline call. + /// + /// Extracts the ordered item list from a deserialized page envelope. + /// + /// + /// Given the deserialized page, the raw response (before disposal), and the current request, + /// returns the next request to send, or to end iteration. + /// + /// + /// Maximum number of pages to fetch. means no limit. + /// + /// + /// A lazy that fetches exactly one page per consumer advance. + /// + public static AsyncPageable Create( + HttpPipeline pipeline, + Request first, + ISerde serde, + DexpaceClientOptions options, + Func> selectItems, + Func nextRequest, + int? maxPages = null) + { + ArgumentNullException.ThrowIfNull(pipeline); + ArgumentNullException.ThrowIfNull(first); + ArgumentNullException.ThrowIfNull(serde); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(selectItems); + ArgumentNullException.ThrowIfNull(nextRequest); + + return new PipelinePageable(pipeline, first, serde, options, selectItems, nextRequest, maxPages); + } + + // ── internal implementation ──────────────────────────────────────────────────────────────── + + private sealed class PipelinePageable( + HttpPipeline pipeline, + Request first, + ISerde serde, + DexpaceClientOptions options, + Func> selectItems, + Func nextRequest, + int? maxPages) : AsyncPageable + { + /// + public override IAsyncEnumerable> AsPages(int? pageSizeHint = null) => + PagesCore(CancellationToken.None); + + /// + public override IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => + ItemsCore(cancellationToken).GetAsyncEnumerator(cancellationToken); + + // Page iterator — fetches one HTTP page per yield. + private async IAsyncEnumerable> PagesCore( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var current = first; + var fetched = 0; + + while (true) + { + if (maxPages.HasValue && fetched >= maxPages.Value) + { + yield break; + } + + var response = await pipeline.SendAsync(current, options, cancellationToken) + .ConfigureAwait(false); + + TPage page; + Status status; + Headers headers; + Request? next; + + try + { + page = await response.Body + .ReadValueAsync(serde, cancellationToken) + .ConfigureAwait(false) + ?? throw new InvalidOperationException( + $"Serde returned null when deserializing page type '{typeof(TPage).FullName}'. " + + "The page deserialization must produce a non-null value."); + + status = response.Status; + headers = response.Headers; + + // Capture next while the response (and its headers) are still alive. + next = nextRequest(page, response, current); + } + finally + { + await response.DisposeAsync().ConfigureAwait(false); + } + + fetched++; + yield return new Page(selectItems(page), status, headers); + + if (next is null) + { + yield break; + } + + current = next; + } + } + + // Item iterator — flattens PagesCore. + private async IAsyncEnumerable ItemsCore( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var page in PagesCore(cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + foreach (var item in page.Values) + { + yield return item; + } + } + } + } +} diff --git a/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs b/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs new file mode 100644 index 0000000..701a651 --- /dev/null +++ b/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs @@ -0,0 +1,500 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +using System.Buffers; +using Dexpace.Sdk.Core.Client; +using Dexpace.Sdk.Core.Configuration; +using Dexpace.Sdk.Core.Http.Common; +using Dexpace.Sdk.Core.Http.Request; +using Dexpace.Sdk.Core.Http.Response; +using Dexpace.Sdk.Core.Pagination; +using Dexpace.Sdk.Core.Pipeline; +using Dexpace.Sdk.Core.Serialization; +using Xunit; + +namespace Dexpace.Sdk.Core.Tests.Pagination; + +/// +/// Integration tests for / +/// / . +/// +public class PageableTests +{ + // ── helpers ──────────────────────────────────────────────────────────────────────────────── + + private static DexpaceClientOptions DefaultOptions => new(); + + // A simple page envelope for tests. + private sealed record TestPage(IReadOnlyList Items, bool HasNext); + + // Builds a pipeline + scripted transport from a sequence of pre-canned responses. + private static (HttpPipeline, ScriptedTransport) MakePipeline(params Response[] responses) + { + var transport = new ScriptedTransport(responses); + var pipeline = new PipelineBuilder().Build(transport); + return (pipeline, transport); + } + + // nextRequest: advance to the next URL when HasNext is true. + private static Request? NextRequest(TestPage page, Response _, Request current) => + page.HasNext ? current with { Url = new Uri(current.Url + "/next") } : null; + + // Convenience: create a pageable over TestPage with int items. + private static AsyncPageable MakePageable( + HttpPipeline pipeline, + ISerde serde, + int? maxPages = null) => + Pageable.Create( + pipeline, + Request.Get("https://api.example.com/items"), + serde, + DefaultOptions, + p => p.Items, + NextRequest, + maxPages); + + // ── scripted transport ───────────────────────────────────────────────────────────────────── + + private sealed class ScriptedTransport(params Response[] responses) : IAsyncHttpClient + { + private int _index; + + public int CallCount => _index; + + public Task ExecuteAsync(Request request, CancellationToken cancellationToken = default) + { + if (_index >= responses.Length) + { + throw new InvalidOperationException( + $"ScriptedTransport exhausted: {responses.Length} response(s) scripted, call #{_index + 1} received."); + } + + return Task.FromResult(responses[_index++]); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + // ── scripted serde ───────────────────────────────────────────────────────────────────────── + + // A fake ISerde that ignores the stream and returns scripted page objects. + private sealed class ScriptedSerde(params TScripted[] pages) : ISerde + { + private int _index; + + public MediaType DefaultMediaType => MediaType.Of("application", "json"); + + public ValueTask SerializeAsync(Stream destination, TVal value, CancellationToken ct = default) => + ValueTask.CompletedTask; + + public async ValueTask DeserializeAsync(Stream source, CancellationToken ct = default) + { + // Consume the stream to avoid leak warnings. + await source.CopyToAsync(Stream.Null, ct).ConfigureAwait(false); + + if (typeof(TVal) != typeof(TScripted)) + { + throw new InvalidOperationException( + $"ScriptedSerde<{typeof(TScripted).Name}> asked for {typeof(TVal).Name}."); + } + + if (_index >= pages.Length) + { + throw new InvalidOperationException("ScriptedSerde exhausted."); + } + + return (TVal)(object)pages[_index++]!; + } + + public void Serialize(IBufferWriter destination, TVal value) { } + + public TVal? Deserialize(ReadOnlySpan utf8) => default; + } + + // ── tracking response body (asserts disposal) ───────────────────────────────────────────── + + private sealed class TrackingBody : ResponseBody + { + private int _consumed; + + public bool Disposed { get; private set; } + + public override MediaType? ContentType => null; + + public override Task OpenReadAsync(CancellationToken cancellationToken = default) + { + if (System.Threading.Interlocked.Exchange(ref _consumed, 1) != 0) + { + throw new Errors.StreamConsumedException("Already consumed."); + } + + return Task.FromResult(new MemoryStream(Array.Empty(), writable: false)); + } + + public override void Dispose() + { + Disposed = true; + base.Dispose(); + } + + public override ValueTask DisposeAsync() + { + Disposed = true; + return base.DisposeAsync(); + } + } + + // ── item flattening ──────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task GetAsyncEnumerator_FlattensItemsAcrossPages() + { + var page1 = new TestPage([1, 2], HasNext: true); + var page2 = new TestPage([3, 4], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, _) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + var items = new List(); + await foreach (var item in pageable) + { + items.Add(item); + } + + Assert.Equal([1, 2, 3, 4], items); + } + + // ── AsPages ──────────────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task AsPages_YieldsCorrectNumberOfPages() + { + var page1 = new TestPage([10, 20], HasNext: true); + var page2 = new TestPage([30], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + var count = 0; + await foreach (var _ in pageable.AsPages()) + { + count++; + } + + Assert.Equal(2, count); + Assert.Equal(2, transport.CallCount); + } + + [Fact] + public async Task AsPages_PageValuesMatchSelectItems() + { + var page1 = new TestPage([7, 8, 9], HasNext: false); + + var serde = new ScriptedSerde(page1); + var (pipeline, _) = MakePipeline(new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + var pages = new List>(); + await foreach (var page in pageable.AsPages()) + { + pages.Add(page); + } + + Assert.Single(pages); + Assert.Equal([7, 8, 9], pages[0].Values); + } + + [Fact] + public async Task AsPages_ExposesStatusAndHeaders() + { + var responseHeaders = Headers.Empty.With("X-Page", "42"); + var page1 = new TestPage([1], HasNext: false); + + var serde = new ScriptedSerde(page1); + var (pipeline, _) = MakePipeline(new Response(Status.Ok, responseHeaders)); + + var pageable = MakePageable(pipeline, serde); + + var pages = new List>(); + await foreach (var page in pageable.AsPages()) + { + pages.Add(page); + } + + Assert.Single(pages); + Assert.Equal(Status.Ok, pages[0].Status); + Assert.Equal( + responseHeaders.GetAll("X-Page"), + pages[0].Headers.GetAll("X-Page")); + } + + // ── laziness ─────────────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task Laziness_AfterFirstPageConsumed_TransportCalledOnce() + { + var page1 = new TestPage([1, 2], HasNext: true); + var page2 = new TestPage([3, 4], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + // Consume only the first page via AsPages enumerator. + await using var enumerator = pageable.AsPages().GetAsyncEnumerator(); + var moved = await enumerator.MoveNextAsync(); + + Assert.True(moved); + // Only one HTTP call should have been made — the second page must not be pre-fetched. + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task Laziness_AdvancingToSecondPage_TriggersSecondSend() + { + var page1 = new TestPage([1], HasNext: true); + var page2 = new TestPage([2], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + await using var enumerator = pageable.AsPages().GetAsyncEnumerator(); + + await enumerator.MoveNextAsync(); // fetches page 1 → 1 call + Assert.Equal(1, transport.CallCount); + + await enumerator.MoveNextAsync(); // fetches page 2 → 2 calls + Assert.Equal(2, transport.CallCount); + } + + // ── maxPages ─────────────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task MaxPages_CapsNumberOfPagesFetched() + { + // Script 5 pages; cap at 2. + var scriptedPages = Enumerable.Range(0, 5) + .Select(i => new TestPage([i], HasNext: true)) + .ToArray(); + + var serde = new ScriptedSerde(scriptedPages); + var responses = Enumerable.Range(0, 5).Select(_ => new Response(Status.Ok)).ToArray(); + var (pipeline, transport) = MakePipeline(responses); + + var pageable = MakePageable(pipeline, serde, maxPages: 2); + + var collected = new List(); + await foreach (var item in pageable) + { + collected.Add(item); + } + + Assert.Equal(2, transport.CallCount); + Assert.Equal([0, 1], collected); + } + + [Fact] + public async Task MaxPages_WhenNull_IteratesAllPages() + { + var page1 = new TestPage([1], HasNext: true); + var page2 = new TestPage([2], HasNext: true); + var page3 = new TestPage([3], HasNext: false); + + var serde = new ScriptedSerde(page1, page2, page3); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde, maxPages: null); + + var items = new List(); + await foreach (var item in pageable) + { + items.Add(item); + } + + Assert.Equal(3, transport.CallCount); + Assert.Equal([1, 2, 3], items); + } + + // ── nextRequest returning null ends iteration ────────────────────────────────────────────── + + [Fact] + public async Task NextRequestReturningNull_EndsIteration() + { + var page1 = new TestPage([1, 2], HasNext: false); // HasNext=false → nextRequest returns null + + var serde = new ScriptedSerde(page1); + var (pipeline, transport) = MakePipeline(new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + var items = new List(); + await foreach (var item in pageable) + { + items.Add(item); + } + + Assert.Equal(1, transport.CallCount); + Assert.Equal([1, 2], items); + } + + // ── response disposal ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task EachResponse_IsDisposedAfterPageYield() + { + var body1 = new TrackingBody(); + var body2 = new TrackingBody(); + + var page1 = new TestPage([1], HasNext: true); + var page2 = new TestPage([2], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, _) = MakePipeline( + new Response(Status.Ok, body: body1), + new Response(Status.Ok, body: body2)); + + var pageable = MakePageable(pipeline, serde); + + await foreach (var _ in pageable) { } + + Assert.True(body1.Disposed, "First response body should be disposed."); + Assert.True(body2.Disposed, "Second response body should be disposed."); + } + + // ── Page constructor ──────────────────────────────────────────────────────────────────── + + [Fact] + public void Page_Constructor_SetsProperties() + { + var values = new List { 1, 2, 3 }; + var status = Status.Ok; + var headers = Headers.Empty.With("X-Foo", "bar"); + + var page = new Page(values, status, headers); + + Assert.Same(values, page.Values); + Assert.Equal(status, page.Status); + Assert.Same(headers, page.Headers); + } + + [Fact] + public void Page_Constructor_NullValues_Throws() + { + Assert.Throws(() => new Page(null!, Status.Ok, Headers.Empty)); + } + + [Fact] + public void Page_Constructor_NullHeaders_Throws() + { + Assert.Throws(() => new Page(Array.Empty(), Status.Ok, null!)); + } + + // ── Pageable.Create argument guards ─────────────────────────────────────────────────────── + + [Fact] + public void Create_NullPipeline_Throws() + { + var serde = new ScriptedSerde(); + Assert.Throws(() => + Pageable.Create( + null!, + Request.Get("https://x.com"), + serde, + DefaultOptions, + p => p.Items, + NextRequest)); + } + + [Fact] + public void Create_NullRequest_Throws() + { + var (pipeline, _) = MakePipeline(); + var serde = new ScriptedSerde(); + Assert.Throws(() => + Pageable.Create( + pipeline, + null!, + serde, + DefaultOptions, + p => p.Items, + NextRequest)); + } + + [Fact] + public void Create_NullSerde_Throws() + { + var (pipeline, _) = MakePipeline(); + Assert.Throws(() => + Pageable.Create( + pipeline, + Request.Get("https://x.com"), + null!, + DefaultOptions, + p => p.Items, + NextRequest)); + } + + [Fact] + public void Create_NullOptions_Throws() + { + var (pipeline, _) = MakePipeline(); + var serde = new ScriptedSerde(); + Assert.Throws(() => + Pageable.Create( + pipeline, + Request.Get("https://x.com"), + serde, + null!, + p => p.Items, + NextRequest)); + } + + [Fact] + public void Create_NullSelectItems_Throws() + { + var (pipeline, _) = MakePipeline(); + var serde = new ScriptedSerde(); + Assert.Throws(() => + Pageable.Create( + pipeline, + Request.Get("https://x.com"), + serde, + DefaultOptions, + null!, + NextRequest)); + } + + [Fact] + public void Create_NullNextRequest_Throws() + { + var (pipeline, _) = MakePipeline(); + var serde = new ScriptedSerde(); + Assert.Throws(() => + Pageable.Create( + pipeline, + Request.Get("https://x.com"), + serde, + DefaultOptions, + p => p.Items, + null!)); + } +} From 11fcc3a01a2eaa78c60081d36204f80ef605f94a Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Tue, 16 Jun 2026 18:33:30 +0300 Subject: [PATCH 2/4] test: cover pager disposal, re-enumeration, and cancellation Add four test scenarios to PageableTests that were missing coverage: early-break disposal (first body disposed, transport called once), exception-path disposal (serde throws but body is still disposed), re-enumeration (each iteration restarts from first, independent call count), and already-cancelled token (items + pages paths both surface OperationCanceledException before the transport is driven). The last scenario required a small correctness fix to PagesCore: add ThrowIfCancellationRequested() at the top of each iteration so a pre-cancelled token is observed before SendAsync is called. Also adds a one-line XML remarks note on the AsPages override stating that pageSizeHint is not plumbed in v1 and that cancellation for the pages path is passed via WithCancellation(token). Co-Authored-By: Claude Sonnet 4.6 --- src/Dexpace.Sdk.Core/Pagination/Pageable.cs | 6 + .../Pagination/PageableTests.cs | 141 ++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/src/Dexpace.Sdk.Core/Pagination/Pageable.cs b/src/Dexpace.Sdk.Core/Pagination/Pageable.cs index 346471b..3d38209 100644 --- a/src/Dexpace.Sdk.Core/Pagination/Pageable.cs +++ b/src/Dexpace.Sdk.Core/Pagination/Pageable.cs @@ -71,6 +71,10 @@ private sealed class PipelinePageable( int? maxPages) : AsyncPageable { /// + /// + /// is not plumbed into the outgoing request in v1; cancel + /// the pages path via .WithCancellation(token) on the returned sequence. + /// public override IAsyncEnumerable> AsPages(int? pageSizeHint = null) => PagesCore(CancellationToken.None); @@ -87,6 +91,8 @@ private async IAsyncEnumerable> PagesCore( while (true) { + cancellationToken.ThrowIfCancellationRequested(); + if (maxPages.HasValue && fetched >= maxPages.Value) { yield break; diff --git a/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs b/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs index 701a651..81f7c5a 100644 --- a/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs +++ b/tests/Dexpace.Sdk.Core.Tests/Pagination/PageableTests.cs @@ -497,4 +497,145 @@ public void Create_NullNextRequest_Throws() p => p.Items, null!)); } + + // ── disposal: early break ───────────────────────────────────────────────────────────────── + + [Fact] + public async Task EarlyBreak_FirstBodyDisposed_AndTransportCalledOnce() + { + var body1 = new TrackingBody(); + var page1 = new TestPage([1, 2], HasNext: true); + var page2 = new TestPage([3, 4], HasNext: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok, body: body1), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + // Consume only the first item then break — second page must never be fetched. + await foreach (var _ in pageable) + { + break; + } + + Assert.True(body1.Disposed, "First response body should be disposed after early break."); + Assert.Equal(1, transport.CallCount); + } + + // ── disposal: exception path ────────────────────────────────────────────────────────────── + + [Fact] + public async Task ExceptionFromSerde_BodyStillDisposed() + { + var body = new TrackingBody(); + + // Serde throws on the first call. + var throwingSerde = new ThrowingOnFirstCallSerde(); + var (pipeline, _) = MakePipeline(new Response(Status.Ok, body: body)); + + var pageable = Pageable.Create( + pipeline, + Request.Get("https://api.example.com/items"), + throwingSerde, + DefaultOptions, + p => p.Items, + NextRequest); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in pageable) { } + }); + + Assert.True(body.Disposed, "Response body must be disposed even when the serde throws."); + } + + // ── re-enumeration ──────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task ReEnumeration_EachEnumerationRestartsFromFirst() + { + // Two passes, each should see the same items and drive the transport independently. + var p1 = new TestPage([10, 20], HasNext: false); + var p2 = new TestPage([10, 20], HasNext: false); // scripted twice + + var serde = new ScriptedSerde(p1, p2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var pageable = MakePageable(pipeline, serde); + + var first = new List(); + await foreach (var item in pageable) { first.Add(item); } + + var second = new List(); + await foreach (var item in pageable) { second.Add(item); } + + Assert.Equal([10, 20], first); + Assert.Equal([10, 20], second); + // Each enumeration should have caused exactly one HTTP call (2 total). + Assert.Equal(2, transport.CallCount); + } + + // ── cancellation ────────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task Cancellation_AlreadyCancelled_Items_ThrowsBeforeTransport() + { + var (pipeline, transport) = MakePipeline(new Response(Status.Ok)); + var serde = new ScriptedSerde(new TestPage([1], HasNext: false)); + var pageable = MakePageable(pipeline, serde); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in pageable.WithCancellation(cts.Token)) { } + }); + + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task Cancellation_AlreadyCancelled_Pages_ThrowsBeforeTransport() + { + var (pipeline, transport) = MakePipeline(new Response(Status.Ok)); + var serde = new ScriptedSerde(new TestPage([1], HasNext: false)); + var pageable = MakePageable(pipeline, serde); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in pageable.AsPages().WithCancellation(cts.Token)) { } + }); + + Assert.Equal(0, transport.CallCount); + } + + // ── helpers for the new tests ───────────────────────────────────────────────────────────── + + // ISerde that always throws InvalidOperationException from DeserializeAsync. + private sealed class ThrowingOnFirstCallSerde : ISerde + { + public MediaType DefaultMediaType => MediaType.Of("application", "json"); + + public ValueTask SerializeAsync(Stream destination, TVal value, CancellationToken ct = default) => + ValueTask.CompletedTask; + + public async ValueTask DeserializeAsync(Stream source, CancellationToken ct = default) + { + // Drain the body before throwing so disposal tracking stays clean. + await source.CopyToAsync(Stream.Null, ct).ConfigureAwait(false); + throw new InvalidOperationException("Serde failure injected by test."); + } + + public void Serialize(System.Buffers.IBufferWriter destination, TVal value) { } + + public TVal? Deserialize(ReadOnlySpan utf8) => default; + } } From 751b1e0b9c533d02f81beb9d72270c36347956d0 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Tue, 16 Jun 2026 18:37:30 +0300 Subject: [PATCH 3/4] feat: add pagination strategy helpers (cursor, page-number, link-header) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add public static class PaginationStrategies with three factory methods: - Cursor: reads a cursor string from the deserialized page and sets it as a URL query parameter on the next request; returns null when the cursor is absent or empty to end iteration. - PageNumber: reads the current integer value of a query parameter (defaults to 1 if absent), increments it, and returns the next request while hasMore(page) is true. - LinkHeader: parses the Link response header per RFC 8288 (comma-separated ; rel="…" entries, handles top-level commas inside angle brackets), resolves the matching URL against the current request URL via Uri.TryCreate, and stops on parse failure or missing rel. Default rel is "next". All helpers are BCL-only (no new package references). Internal query manipulation uses a zero-allocation ref-struct QueryPairEnumerator and UriBuilder. Each strategy produces current with { Url = newUrl }, preserving method, headers, and body. Covered by PaginationStrategiesTests: cursor set/stop/replace, cursor preserves method+headers, page-number increment/stop/absent-default, link-header next/stop/relative/missing/multi-entry/malformed/default-rel, and one end-to-end Pageable.Create + cursor strategy two-page walk. Co-Authored-By: Claude Sonnet 4.6 --- .../Pagination/PaginationStrategies.cs | 367 ++++++++++++++++++ .../Pagination/PaginationStrategiesTests.cs | 357 +++++++++++++++++ 2 files changed, 724 insertions(+) create mode 100644 src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs create mode 100644 tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs diff --git a/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs b/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs new file mode 100644 index 0000000..bfa52b8 --- /dev/null +++ b/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs @@ -0,0 +1,367 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +using System.Globalization; +using Dexpace.Sdk.Core.Http.Request; +using Dexpace.Sdk.Core.Http.Response; + +namespace Dexpace.Sdk.Core.Pagination; + +/// +/// Factory methods that build nextRequest delegates for use with +/// . +/// +/// +/// Each factory returns a compatible with the +/// nextRequest parameter of . The returned delegate +/// receives the deserialized page, the raw (not-yet-disposed) response, and the current request, +/// and must return the next to send or to stop +/// iteration. +/// +public static class PaginationStrategies +{ + // ── Cursor ──────────────────────────────────────────────────────────────────────────────── + + /// + /// Builds a next-request delegate that drives cursor-based pagination. + /// + /// The deserialized page-envelope type. + /// + /// Extracts the continuation cursor from a deserialized page. Return + /// or empty string to end iteration. + /// + /// + /// The URL query-string key to set to the cursor value on the next request. + /// + /// + /// A delegate that returns the next with + /// set to the cursor, or when the cursor is absent. + /// + public static Func Cursor( + Func nextCursor, + string queryParameter) + { + ArgumentNullException.ThrowIfNull(nextCursor); + ArgumentNullException.ThrowIfNull(queryParameter); + + return (page, _, current) => + { + var cursor = nextCursor(page); + if (string.IsNullOrEmpty(cursor)) + { + return null; + } + + return current with { Url = SetQueryParameter(current.Url, queryParameter, cursor) }; + }; + } + + // ── PageNumber ──────────────────────────────────────────────────────────────────────────── + + /// + /// Builds a next-request delegate that drives page-number pagination. + /// + /// The deserialized page-envelope type. + /// + /// The URL query-string key that carries the page number. If absent on the current request URL + /// the current page number is treated as 1. + /// + /// + /// Returns when the response contains more pages to fetch. + /// + /// + /// A delegate that increments the page-number query parameter and returns the next + /// , or when returns + /// . + /// + public static Func PageNumber( + string queryParameter, + Func hasMore) + { + ArgumentNullException.ThrowIfNull(queryParameter); + ArgumentNullException.ThrowIfNull(hasMore); + + return (page, _, current) => + { + if (!hasMore(page)) + { + return null; + } + + var raw = GetQueryParameter(current.Url, queryParameter); + var currentPage = raw is not null && int.TryParse(raw, out var n) ? n : 1; + var nextPage = currentPage + 1; + return current with { Url = SetQueryParameter(current.Url, queryParameter, nextPage.ToString(CultureInfo.InvariantCulture)) }; + }; + } + + // ── LinkHeader ──────────────────────────────────────────────────────────────────────────── + + /// + /// Builds a next-request delegate that drives link-header pagination (RFC 8288). + /// + /// + /// The rel type to look for in the Link response header. + /// Defaults to "next". + /// + /// + /// A delegate that reads the Link response header, resolves the URL for the + /// given , and returns the next ; or + /// if no matching entry is found or the URL cannot be resolved. + /// + public static Func LinkHeader(string rel = "next") + { + ArgumentNullException.ThrowIfNull(rel); + + return (_, response, current) => + { + var linkHeaderValue = response.Headers.Get("Link"); + if (string.IsNullOrEmpty(linkHeaderValue)) + { + return null; + } + + var linkUrl = ParseLinkHeader(linkHeaderValue, rel); + if (linkUrl is null) + { + return null; + } + + // Resolve the link URL against the current request URL. + if (!Uri.TryCreate(current.Url, linkUrl, out var resolved)) + { + return null; + } + + return current with { Url = resolved }; + }; + } + + // ── private helpers ─────────────────────────────────────────────────────────────────────── + + /// + /// Returns the raw (URI-decoded) value of the first occurrence of + /// in the query string of , or + /// when the key is absent. + /// + private static string? GetQueryParameter(Uri uri, string key) + { + var query = uri.Query; + if (string.IsNullOrEmpty(query)) + { + return null; + } + + // Strip leading '?'. + var span = query.AsSpan().TrimStart('?'); + var encodedKey = Uri.EscapeDataString(key); + + foreach (var pair in new QueryPairEnumerator(span)) + { + var eqIndex = pair.IndexOf('='); + var pairKey = eqIndex < 0 ? pair : pair[..eqIndex]; + if (pairKey.Equals(encodedKey.AsSpan(), StringComparison.OrdinalIgnoreCase)) + { + if (eqIndex < 0) + { + return string.Empty; + } + + return Uri.UnescapeDataString(new string(pair[(eqIndex + 1)..])); + } + } + + return null; + } + + /// + /// Returns a new identical to except that + /// is set to (replacing any existing + /// occurrence, or appending if absent). + /// + private static Uri SetQueryParameter(Uri uri, string key, string value) + { + var encodedKey = Uri.EscapeDataString(key); + var encodedValue = Uri.EscapeDataString(value); + var pair = $"{encodedKey}={encodedValue}"; + + var existing = uri.Query.TrimStart('?'); + string newQuery; + + if (string.IsNullOrEmpty(existing)) + { + newQuery = pair; + } + else + { + // Replace the existing key=value if present, otherwise append. + var span = existing.AsSpan(); + var sb = new System.Text.StringBuilder(); + var replaced = false; + + foreach (var segment in new QueryPairEnumerator(span)) + { + var eqIndex = segment.IndexOf('='); + var segKey = eqIndex < 0 ? segment : segment[..eqIndex]; + + if (!replaced && segKey.Equals(encodedKey.AsSpan(), StringComparison.OrdinalIgnoreCase)) + { + if (sb.Length > 0) { sb.Append('&'); } + sb.Append(pair); + replaced = true; + } + else + { + if (sb.Length > 0) { sb.Append('&'); } + sb.Append(new string(segment)); + } + } + + if (!replaced) + { + if (sb.Length > 0) { sb.Append('&'); } + sb.Append(pair); + } + + newQuery = sb.ToString(); + } + + var builder = new UriBuilder(uri) { Query = newQuery }; + return builder.Uri; + } + + /// + /// Parses the value of a Link header and returns the URL string for the entry + /// whose rel matches (case-insensitive), or + /// if not found. + /// + /// + /// Handles the comma-separated format: + /// <https://api.example.com/items?page=2>; rel="next", <...>; rel="prev" + /// + private static string? ParseLinkHeader(string headerValue, string rel) + { + // Split on commas that are not inside angle brackets. + // We do a simple state-machine split because link URLs may theoretically + // contain commas (e.g., in query parameters), but for RFC 8288 the entries + // are comma-separated at the top level. + var entries = SplitLinkEntries(headerValue); + + foreach (var entry in entries) + { + var trimmed = entry.Trim(); + if (trimmed.Length == 0) { continue; } + + // Each entry: ; param=value; param=value… + // Split on ';' + var parts = trimmed.Split(';'); + if (parts.Length < 2) { continue; } + + // First part must be . + var urlPart = parts[0].Trim(); + if (!urlPart.StartsWith('<') || !urlPart.EndsWith('>')) { continue; } + + var linkUrl = urlPart[1..^1].Trim(); + + // Scan the remaining parts for rel="". + var hasMatchingRel = false; + for (var i = 1; i < parts.Length; i++) + { + var param = parts[i].Trim(); + + // Strip optional quotes and compare. + if (param.StartsWith("rel=", StringComparison.OrdinalIgnoreCase)) + { + var relValue = param["rel=".Length..].Trim().Trim('"'); + if (string.Equals(relValue, rel, StringComparison.OrdinalIgnoreCase)) + { + hasMatchingRel = true; + break; + } + } + } + + if (hasMatchingRel) + { + return linkUrl; + } + } + + return null; + } + + /// + /// Splits a Link header value on top-level commas (outside angle brackets). + /// + private static IEnumerable SplitLinkEntries(string headerValue) + { + var depth = 0; + var start = 0; + + for (var i = 0; i < headerValue.Length; i++) + { + switch (headerValue[i]) + { + case '<': depth++; break; + case '>': if (depth > 0) { depth--; } break; + case ',' when depth == 0: + yield return headerValue[start..i]; + start = i + 1; + break; + } + } + + if (start < headerValue.Length) + { + yield return headerValue[start..]; + } + } + + // ── query-pair ref-struct enumerator ────────────────────────────────────────────────────── + + /// + /// Zero-allocation enumerator over key=value pairs in a query string span + /// (without the leading ?). Each iteration returns a + /// slice covering one key=value segment. + /// + private ref struct QueryPairEnumerator + { + private ReadOnlySpan _remaining; + private ReadOnlySpan _current; + + internal QueryPairEnumerator(ReadOnlySpan query) => _remaining = query; + + /// Returns this so the type works in foreach. + public readonly QueryPairEnumerator GetEnumerator() => this; + + /// The current segment. + public readonly ReadOnlySpan Current => _current; + + /// Advances to the next segment. + public bool MoveNext() + { + while (!_remaining.IsEmpty) + { + var idx = _remaining.IndexOf('&'); + if (idx < 0) + { + _current = _remaining; + _remaining = []; + } + else + { + _current = _remaining[..idx]; + _remaining = _remaining[(idx + 1)..]; + } + + // Skip empty segments (e.g. leading/trailing '&'). + if (!_current.IsEmpty) + { + return true; + } + } + + return false; + } + } +} diff --git a/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs b/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs new file mode 100644 index 0000000..513f668 --- /dev/null +++ b/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs @@ -0,0 +1,357 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +using System.Buffers; +using Dexpace.Sdk.Core.Client; +using Dexpace.Sdk.Core.Configuration; +using Dexpace.Sdk.Core.Http.Common; +using Dexpace.Sdk.Core.Http.Request; +using Dexpace.Sdk.Core.Http.Response; +using Dexpace.Sdk.Core.Pagination; +using Dexpace.Sdk.Core.Pipeline; +using Dexpace.Sdk.Core.Serialization; +using Xunit; + +namespace Dexpace.Sdk.Core.Tests.Pagination; + +/// +/// Unit tests for . +/// +public class PaginationStrategiesTests +{ + // ── helpers ──────────────────────────────────────────────────────────────────────────────── + + private static DexpaceClientOptions DefaultOptions => new(); + + // Simple page envelope. + private sealed record TestPage(IReadOnlyList Items, string? Cursor, bool HasMore); + + private static Request BaseRequest(string url) => Request.Get(url); + + // A no-op response used when the response is not examined by the strategy under test. + private static Response EmptyResponse => new(Status.Ok); + + // A response that carries a Link header. + private static Response ResponseWithLink(string linkHeaderValue) => + new(Status.Ok, Headers.Empty.With("Link", linkHeaderValue)); + + // ── ScriptedTransport + ScriptedSerde (mirrors PageableTests helpers) ────────────────────── + + private sealed class ScriptedTransport(params Response[] responses) : IAsyncHttpClient + { + private int _index; + + public int CallCount => _index; + + public Task ExecuteAsync(Request request, CancellationToken cancellationToken = default) + { + if (_index >= responses.Length) + { + throw new InvalidOperationException( + $"ScriptedTransport exhausted: call #{_index + 1} received."); + } + + return Task.FromResult(responses[_index++]); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class ScriptedSerde(params TScripted[] pages) : ISerde + { + private int _index; + + public MediaType DefaultMediaType => MediaType.Of("application", "json"); + + public ValueTask SerializeAsync(Stream destination, TVal value, CancellationToken ct = default) => + ValueTask.CompletedTask; + + public async ValueTask DeserializeAsync(Stream source, CancellationToken ct = default) + { + await source.CopyToAsync(Stream.Null, ct).ConfigureAwait(false); + + if (typeof(TVal) != typeof(TScripted)) + { + throw new InvalidOperationException( + $"ScriptedSerde<{typeof(TScripted).Name}> asked for {typeof(TVal).Name}."); + } + + if (_index >= pages.Length) + { + throw new InvalidOperationException("ScriptedSerde exhausted."); + } + + return (TVal)(object)pages[_index++]!; + } + + public void Serialize(IBufferWriter destination, TVal value) { } + + public TVal? Deserialize(ReadOnlySpan utf8) => default; + } + + // Builds a pipeline from a scripted transport. + private static (HttpPipeline, ScriptedTransport) MakePipeline(params Response[] responses) + { + var transport = new ScriptedTransport(responses); + var pipeline = new PipelineBuilder().Build(transport); + return (pipeline, transport); + } + + // ── Cursor ──────────────────────────────────────────────────────────────────────────────── + + [Fact] + public void Cursor_WithNonNullCursor_SetsQueryParameter() + { + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var current = BaseRequest("https://api.example.com/items"); + var page = new TestPage([], "abc123", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("abc123", GetQueryParam(next.Url, "cursor")); + } + + [Fact] + public void Cursor_WithNullCursor_ReturnsNull() + { + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var current = BaseRequest("https://api.example.com/items"); + var page = new TestPage([], null, HasMore: false); + + var next = strategy(page, EmptyResponse, current); + + Assert.Null(next); + } + + [Fact] + public void Cursor_WithEmptyStringCursor_ReturnsNull() + { + var strategy = PaginationStrategies.Cursor(p => string.Empty, "cursor"); + var current = BaseRequest("https://api.example.com/items"); + var page = new TestPage([], string.Empty, HasMore: false); + + var next = strategy(page, EmptyResponse, current); + + Assert.Null(next); + } + + [Fact] + public void Cursor_PreservesMethodAndHeaders() + { + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "after"); + var current = BaseRequest("https://api.example.com/items") + .WithHeader("Authorization", "Bearer tok"); + var page = new TestPage([], "tok_next", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal(current.Method, next.Method); + Assert.Equal("Bearer tok", next.Headers.Get("Authorization")); + } + + [Fact] + public void Cursor_ReplacesExistingCursorOnSubsequentPages() + { + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var current = BaseRequest("https://api.example.com/items?cursor=old"); + var page = new TestPage([], "new_cursor", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("new_cursor", GetQueryParam(next.Url, "cursor")); + // Old value must not appear. + Assert.DoesNotContain("old", next.Url.Query); + } + + // ── PageNumber ──────────────────────────────────────────────────────────────────────────── + + [Fact] + public void PageNumber_WhenHasMore_IncrementsPageParameter() + { + var strategy = PaginationStrategies.PageNumber("page", p => p.HasMore); + var current = BaseRequest("https://api.example.com/items?page=2"); + var page = new TestPage([], null, HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("3", GetQueryParam(next.Url, "page")); + } + + [Fact] + public void PageNumber_WhenPageParameterAbsent_DefaultsToOneAndIncrements() + { + var strategy = PaginationStrategies.PageNumber("page", p => p.HasMore); + var current = BaseRequest("https://api.example.com/items"); + var page = new TestPage([], null, HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("2", GetQueryParam(next.Url, "page")); + } + + [Fact] + public void PageNumber_WhenNoMore_ReturnsNull() + { + var strategy = PaginationStrategies.PageNumber("page", p => p.HasMore); + var current = BaseRequest("https://api.example.com/items?page=3"); + var page = new TestPage([], null, HasMore: false); + + var next = strategy(page, EmptyResponse, current); + + Assert.Null(next); + } + + // ── LinkHeader ──────────────────────────────────────────────────────────────────────────── + + [Fact] + public void LinkHeader_WithNextRel_ReturnsAbsoluteUrl() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink("; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=2"), next.Url); + } + + [Fact] + public void LinkHeader_WithNoMatchingRel_ReturnsNull() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink("; rel=\"prev\""); + + var next = strategy(default!, response, current); + + Assert.Null(next); + } + + [Fact] + public void LinkHeader_WithRelativeUrl_ResolvesAgainstCurrentRequest() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + // Relative URL — should be resolved against current request URL. + var response = ResponseWithLink("; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=2"), next.Url); + } + + [Fact] + public void LinkHeader_WithMissingLinkHeader_ReturnsNull() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + + var next = strategy(default!, EmptyResponse, current); + + Assert.Null(next); + } + + [Fact] + public void LinkHeader_WithMultipleRelEntries_FindsNext() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink( + "; rel=\"prev\", ; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=3"), next.Url); + } + + [Fact] + public void LinkHeader_WithMalformedEntry_ReturnsNull() + { + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + // No angle brackets — malformed. + var response = ResponseWithLink("not-a-valid-link-header"); + + var next = strategy(default!, response, current); + + Assert.Null(next); + } + + [Fact] + public void LinkHeader_DefaultRelIsNext() + { + // No rel= parameter — use default. + var strategy = PaginationStrategies.LinkHeader(); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink("; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + } + + // ── End-to-end: Pageable.Create + a strategy ────────────────────────────────────────────── + + [Fact] + public async Task EndToEnd_CursorStrategy_TwoPagesViaFakePipeline() + { + var page1 = new TestPage([1, 2], Cursor: "cur1", HasMore: true); + var page2 = new TestPage([3, 4], Cursor: null, HasMore: false); + + var serde = new ScriptedSerde(page1, page2); + var (pipeline, transport) = MakePipeline( + new Response(Status.Ok), + new Response(Status.Ok)); + + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + + var pageable = Pageable.Create( + pipeline, + Request.Get("https://api.example.com/items"), + serde, + DefaultOptions, + p => p.Items, + strategy); + + var items = new List(); + await foreach (var item in pageable) + { + items.Add(item); + } + + Assert.Equal([1, 2, 3, 4], items); + Assert.Equal(2, transport.CallCount); + } + + // ── helpers ──────────────────────────────────────────────────────────────────────────────── + + /// + /// Extracts the raw (decoded) value of from 's + /// query string, or if absent. + /// + private static string? GetQueryParam(Uri uri, string key) + { + var query = uri.Query.TrimStart('?'); + if (string.IsNullOrEmpty(query)) { return null; } + + foreach (var pair in query.Split('&')) + { + var eq = pair.IndexOf('=', StringComparison.Ordinal); + var pairKey = eq < 0 ? pair : pair[..eq]; + if (string.Equals(Uri.UnescapeDataString(pairKey), key, StringComparison.OrdinalIgnoreCase)) + { + return eq < 0 ? string.Empty : Uri.UnescapeDataString(pair[(eq + 1)..]); + } + } + + return null; + } +} From 369f3ae3c55dcfe691bc80103e87e6e2d7e3f73c Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Tue, 16 Jun 2026 18:50:44 +0300 Subject: [PATCH 4/4] fix: harden Link-header parsing (quoted commas, multi-valued rel, scheme guard) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three correctness issues in PaginationStrategies.LinkHeader: 1. Quote-aware splitting — the top-level comma split (SplitLinkEntries) and the per-entry semicolon split (new SplitLinkParams) now suppress splitting while inside a double-quoted run, so a comma or semicolon inside a param value like title=\"Page 3, final\" no longer breaks the entry and stops pagination early. 2. Multi-valued rel — RFC 8288 specifies rel as a space-separated token list. The rel value is now split on whitespace and any matching token (case-insensitive) is accepted, so rel=\"prev next\" correctly matches a \"next\" lookup. 3. Scheme guard — after resolving the link URL against the current request URL, returns null unless the resolved URI is absolute with an http or https scheme. This prevents mailto:, ftp:, javascript:, etc. from reaching the request layer, where current with { Url = ... } would bypass Request's constructor validation. Nine new tests added covering all three fixes plus fragment/sibling-param preservation and body carryover for both Cursor and PageNumber strategies. Co-Authored-By: Claude Opus 4.8 --- .../Pagination/PaginationStrategies.cs | 111 +++++++++++--- .../Pagination/PaginationStrategiesTests.cs | 139 ++++++++++++++++++ 2 files changed, 228 insertions(+), 22 deletions(-) diff --git a/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs b/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs index bfa52b8..cc3425b 100644 --- a/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs +++ b/src/Dexpace.Sdk.Core/Pagination/PaginationStrategies.cs @@ -133,6 +133,17 @@ public static class PaginationStrategies return null; } + // Scheme guard: only allow http/https to prevent untrusted Link headers + // (mailto:, javascript:, ftp:, etc.) from producing a request with a + // non-http/https URL. Note: `current with { Url = ... }` bypasses the + // Request constructor's scheme validation, so the guard must live here. + if (!resolved.IsAbsoluteUri + || (!resolved.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !resolved.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))) + { + return null; + } + return current with { Url = resolved }; }; } @@ -236,15 +247,22 @@ private static Uri SetQueryParameter(Uri uri, string key, string value) /// if not found. /// /// + /// /// Handles the comma-separated format: /// <https://api.example.com/items?page=2>; rel="next", <...>; rel="prev" + /// + /// + /// Quoted parameter values (e.g. title="Page 3, final") are handled correctly: + /// commas and semicolons inside a double-quoted run are not treated as delimiters. + /// + /// + /// Per RFC 8288, rel is a space-separated token list; any token that matches + /// is accepted. + /// /// private static string? ParseLinkHeader(string headerValue, string rel) { - // Split on commas that are not inside angle brackets. - // We do a simple state-machine split because link URLs may theoretically - // contain commas (e.g., in query parameters), but for RFC 8288 the entries - // are comma-separated at the top level. + // Split on commas that are not inside angle brackets or double-quoted strings. var entries = SplitLinkEntries(headerValue); foreach (var entry in entries) @@ -253,9 +271,9 @@ private static Uri SetQueryParameter(Uri uri, string key, string value) if (trimmed.Length == 0) { continue; } // Each entry: ; param=value; param=value… - // Split on ';' - var parts = trimmed.Split(';'); - if (parts.Length < 2) { continue; } + // Use quote-aware split on ';'. + var parts = SplitLinkParams(trimmed); + if (parts.Count < 2) { continue; } // First part must be . var urlPart = parts[0].Trim(); @@ -263,21 +281,26 @@ private static Uri SetQueryParameter(Uri uri, string key, string value) var linkUrl = urlPart[1..^1].Trim(); - // Scan the remaining parts for rel="". + // Scan the remaining parts for rel= (RFC 8288: space-separated token list). var hasMatchingRel = false; - for (var i = 1; i < parts.Length; i++) + for (var i = 1; i < parts.Count; i++) { var param = parts[i].Trim(); - // Strip optional quotes and compare. if (param.StartsWith("rel=", StringComparison.OrdinalIgnoreCase)) { + // Strip optional surrounding quotes, then split the token list on whitespace. var relValue = param["rel=".Length..].Trim().Trim('"'); - if (string.Equals(relValue, rel, StringComparison.OrdinalIgnoreCase)) + foreach (var token in relValue.Split(' ', StringSplitOptions.RemoveEmptyEntries)) { - hasMatchingRel = true; - break; + if (string.Equals(token, rel, StringComparison.OrdinalIgnoreCase)) + { + hasMatchingRel = true; + break; + } } + + if (hasMatchingRel) { break; } } } @@ -291,23 +314,34 @@ private static Uri SetQueryParameter(Uri uri, string key, string value) } /// - /// Splits a Link header value on top-level commas (outside angle brackets). + /// Splits a Link header value on top-level commas — those that are outside + /// both angle brackets (<…>) and double-quoted strings ("…"). /// private static IEnumerable SplitLinkEntries(string headerValue) { - var depth = 0; + var depth = 0; // angle-bracket nesting + var inQuotes = false; // inside "…" var start = 0; for (var i = 0; i < headerValue.Length; i++) { - switch (headerValue[i]) + var ch = headerValue[i]; + + if (ch == '"') { - case '<': depth++; break; - case '>': if (depth > 0) { depth--; } break; - case ',' when depth == 0: - yield return headerValue[start..i]; - start = i + 1; - break; + inQuotes = !inQuotes; + } + else if (!inQuotes) + { + switch (ch) + { + case '<': depth++; break; + case '>': if (depth > 0) { depth--; } break; + case ',' when depth == 0: + yield return headerValue[start..i]; + start = i + 1; + break; + } } } @@ -317,6 +351,39 @@ private static IEnumerable SplitLinkEntries(string headerValue) } } + /// + /// Splits one Link entry on semicolons that are outside double-quoted strings. + /// The first element is always the <URL> target; the remainder are parameters. + /// + private static List SplitLinkParams(string entry) + { + var parts = new List(); + var inQuotes = false; + var start = 0; + + for (var i = 0; i < entry.Length; i++) + { + var ch = entry[i]; + + if (ch == '"') + { + inQuotes = !inQuotes; + } + else if (!inQuotes && ch == ';') + { + parts.Add(entry[start..i]); + start = i + 1; + } + } + + if (start <= entry.Length) + { + parts.Add(entry[start..]); + } + + return parts; + } + // ── query-pair ref-struct enumerator ────────────────────────────────────────────────────── /// diff --git a/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs b/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs index 513f668..86a5d89 100644 --- a/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs +++ b/tests/Dexpace.Sdk.Core.Tests/Pagination/PaginationStrategiesTests.cs @@ -298,6 +298,145 @@ public void LinkHeader_DefaultRelIsNext() Assert.NotNull(next); } + [Fact] + public void LinkHeader_QuotedCommaInParam_DoesNotSplitEntry() + { + // A comma inside a quoted parameter value must not be treated as an entry separator. + // Without quote-aware splitting, the entry is broken at the comma inside the title + // and the rel="next" token ends up in a truncated fragment, so parsing returns null. + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink( + "; rel=\"next\"; title=\"Page 3, final\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=3"), next.Url); + } + + [Fact] + public void LinkHeader_QuotedSemicolonInParam_DoesNotBreakParamParsing() + { + // A semicolon inside a quoted parameter value must not be treated as a param separator. + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink( + "; title=\"A; B\"; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=2"), next.Url); + } + + [Fact] + public void LinkHeader_MultiValuedRel_MatchesWhenAnyTokenEquals() + { + // RFC 8288: rel is a space-separated token list; rel="prev next" should match "next". + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink( + "; rel=\"prev next\""); + + var next = strategy(default!, response, current); + + Assert.NotNull(next); + Assert.Equal(new Uri("https://api.example.com/items?page=2"), next.Url); + } + + [Fact] + public void LinkHeader_MailtoScheme_ReturnsNull() + { + // A Link header with a mailto: URL must not be followed. + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink("; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.Null(next); + } + + [Fact] + public void LinkHeader_FtpScheme_ReturnsNull() + { + // A Link header with an ftp: URL must not be followed. + var strategy = PaginationStrategies.LinkHeader("next"); + var current = BaseRequest("https://api.example.com/items"); + var response = ResponseWithLink("; rel=\"next\""); + + var next = strategy(default!, response, current); + + Assert.Null(next); + } + + // ── SetQueryParameter ───────────────────────────────────────────────────────────────────── + + [Fact] + public void SetQueryParameter_PreservesFragment() + { + // SetQueryParameter must keep an existing fragment. + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var current = BaseRequest("https://api.example.com/items#section2"); + var page = new TestPage([], "abc", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("section2", next.Url.Fragment.TrimStart('#')); + Assert.Equal("abc", GetQueryParam(next.Url, "cursor")); + } + + [Fact] + public void SetQueryParameter_PreservesSiblingQueryParam() + { + // SetQueryParameter must keep unrelated query parameters while replacing/adding the target. + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var current = BaseRequest("https://api.example.com/items?limit=10"); + var page = new TestPage([], "tok", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Equal("tok", GetQueryParam(next.Url, "cursor")); + Assert.Equal("10", GetQueryParam(next.Url, "limit")); + } + + // ── Body preservation ───────────────────────────────────────────────────────────────────── + + [Fact] + public void Cursor_PreservesRequestBody() + { + // A request carrying a body must have that body carried over to the next request. + var strategy = PaginationStrategies.Cursor(p => p.Cursor, "cursor"); + var body = RequestBody.FromBytes(new byte[] { 0x01, 0x02 }, MediaType.Of("application", "octet-stream")); + var current = Request.Create(Method.Post, "https://api.example.com/items", body: body); + var page = new TestPage([], "tok", HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Same(body, next.Body); + Assert.Equal("tok", GetQueryParam(next.Url, "cursor")); + } + + [Fact] + public void PageNumber_PreservesRequestBody() + { + // A request carrying a body must have that body carried over to the next request. + var strategy = PaginationStrategies.PageNumber("page", p => p.HasMore); + var body = RequestBody.FromBytes(new byte[] { 0xAB }, MediaType.Of("application", "octet-stream")); + var current = Request.Create(Method.Post, "https://api.example.com/items?page=1", body: body); + var page = new TestPage([], null, HasMore: true); + + var next = strategy(page, EmptyResponse, current); + + Assert.NotNull(next); + Assert.Same(body, next.Body); + Assert.Equal("2", GetQueryParam(next.Url, "page")); + } + // ── End-to-end: Pageable.Create + a strategy ────────────────────────────────────────────── [Fact]