diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 0000000..29dbd05 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,31 @@ +name: Test + +on: + pull_request: + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node (for contract mock server) + uses: actions/setup-node@v4 + with: + node-version: 20.x + + - name: Fetch contract mock server + run: curl -fsSL -o mock-server.mjs https://raw.githubusercontent.com/prerender/integration-contract/main/mock-server.mjs + + - name: Restore dependencies + run: dotnet restore tests/Prerender.AspNetCore.Tests.csproj + + - name: Run tests + run: dotnet test tests/Prerender.AspNetCore.Tests.csproj --no-restore --verbosity normal diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..edeadce --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +bin/ +obj/ +*.user +mock-server.mjs diff --git a/Prerender.AspNetCore.csproj b/Prerender.AspNetCore.csproj new file mode 100644 index 0000000..507807e --- /dev/null +++ b/Prerender.AspNetCore.csproj @@ -0,0 +1,27 @@ + + + net8.0 + enable + enable + Prerender.AspNetCore + 1.0.0 + Prerender.io + ASP.NET Core middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io + MIT + README.md + https://github.com/prerender/integrations + git + + + + + + + + + + + + + + diff --git a/PrerenderMiddleware.cs b/PrerenderMiddleware.cs new file mode 100644 index 0000000..a4f99a5 --- /dev/null +++ b/PrerenderMiddleware.cs @@ -0,0 +1,111 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Prerender.AspNetCore; + +public class PrerenderMiddleware : IMiddleware +{ + public const string Version = "1.0.0"; + + private static readonly string[] CrawlerUserAgents = + [ + "googlebot", "yahoo", "bingbot", "baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", + "embedly", "quora link preview", "showyoubot", "outbrain", + "pinterest", "slackbot", "w3c_validator", "perplexity", + "oai-searchbot", "chatgpt-user", "gptbot", "claudebot", "amazonbot", + ]; + + private static readonly string[] ExtensionsToIgnore = + [ + ".js", ".css", ".xml", ".less", ".png", ".jpg", ".jpeg", ".gif", + ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", + ".exe", ".wmv", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", + ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", + ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".ttf", ".woff", ".svg", + ]; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly PrerenderOptions _options; + private readonly ILogger _logger; + + public PrerenderMiddleware( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _options = options.Value; + _logger = logger; + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + if (!ShouldPrerender(context)) + { + await next(context); + return; + } + + try + { + var client = _httpClientFactory.CreateClient("prerender"); + var apiUrl = BuildApiUrl(context); + + using var request = new HttpRequestMessage(HttpMethod.Get, apiUrl); + request.Headers.TryAddWithoutValidation( + "User-Agent", context.Request.Headers["User-Agent"].ToString()); + if (!string.IsNullOrWhiteSpace(_options.Token)) + request.Headers.TryAddWithoutValidation("X-Prerender-Token", _options.Token); + request.Headers.TryAddWithoutValidation("X-Prerender-Int-Type", "AspNetCore"); + request.Headers.TryAddWithoutValidation("X-Prerender-Int-Version", Version); + request.Headers.TryAddWithoutValidation("X-Prerender-Request-Id", Guid.NewGuid().ToString()); + + using var response = await client.SendAsync(request, context.RequestAborted); + context.Response.StatusCode = (int)response.StatusCode; + var body = await response.Content.ReadAsStringAsync(); + await context.Response.WriteAsync(body); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "Prerender service unreachable, falling back"); + await next(context); + } + } + + private static bool ShouldPrerender(HttpContext context) + { + if (context.Request.Method != HttpMethods.Get) return false; + + var path = context.Request.Path.Value ?? string.Empty; + if (IsStaticAsset(path)) return false; + + if (context.Request.Query.ContainsKey("_escaped_fragment_")) return true; + if (context.Request.Headers.ContainsKey("X-Bufferbot")) return true; + + var ua = context.Request.Headers["User-Agent"].ToString(); + return !string.IsNullOrEmpty(ua) && IsBot(ua); + } + + private string BuildApiUrl(HttpContext context) + { + var serviceUrl = _options.ServiceUrl.TrimEnd('/') + "/"; + var scheme = context.Request.Scheme; + var host = context.Request.Host.Value; + var pathAndQuery = context.Request.Path + context.Request.QueryString; + return $"{serviceUrl}{scheme}://{host}{pathAndQuery}"; + } + + private static bool IsBot(string userAgent) + { + var ua = userAgent.ToLowerInvariant(); + return CrawlerUserAgents.Any(bot => ua.Contains(bot)); + } + + private static bool IsStaticAsset(string path) + { + var lower = path.ToLowerInvariant(); + return ExtensionsToIgnore.Any(ext => lower.EndsWith(ext)); + } +} diff --git a/PrerenderOptions.cs b/PrerenderOptions.cs new file mode 100644 index 0000000..715f494 --- /dev/null +++ b/PrerenderOptions.cs @@ -0,0 +1,7 @@ +namespace Prerender.AspNetCore; + +public class PrerenderOptions +{ + public string? Token { get; set; } + public string ServiceUrl { get; set; } = "https://service.prerender.io/"; +} diff --git a/PrerenderServiceExtensions.cs b/PrerenderServiceExtensions.cs new file mode 100644 index 0000000..30ac9d8 --- /dev/null +++ b/PrerenderServiceExtensions.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Prerender.AspNetCore; + +public static class PrerenderServiceExtensions +{ + public static IServiceCollection AddPrerender(this IServiceCollection services) + { + services.AddOptions().BindConfiguration("Prerender"); + services.AddHttpClient("prerender"); + services.AddTransient(); + return services; + } + + public static IApplicationBuilder UsePrerender(this IApplicationBuilder app) + => app.UseMiddleware(); +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5b6aac --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# prerender-aspnetcore + +ASP.NET Core middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers. + +Compatible with **ASP.NET Core 8+** and **.NET 8+**. + +## Installation + +```bash +dotnet add package Prerender.AspNetCore +``` + +## Setup + +Register the middleware in `Program.cs`: + +```csharp +builder.Services.AddPrerender(); + +var app = builder.Build(); +app.UsePrerender(); // place before routing middleware +``` + +Add your token to `appsettings.json`: + +```json +{ + "Prerender": { + "Token": "YOUR_PRERENDER_TOKEN" + } +} +``` + +The middleware must be placed **before** routing to intercept bot requests early. + +## Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| `Prerender:Token` | `null` | Your Prerender.io token | +| `Prerender:ServiceUrl` | `https://service.prerender.io/` | Prerender service URL (override for self-hosted Prerender) | + +## Self-hosted Prerender + +```json +{ + "Prerender": { + "ServiceUrl": "http://your-prerender-server:3000" + } +} +``` + +## How it works + +Requests are prerendered when **all** of the following are true: + +- The HTTP method is `GET` +- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.) + — OR the URL contains `_escaped_fragment_` + — OR the `X-Bufferbot` header is present +- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.) + +Everything else passes through to your normal ASP.NET Core pipeline. + +If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response. + +## License + +MIT diff --git a/tests/Prerender.AspNetCore.Tests.csproj b/tests/Prerender.AspNetCore.Tests.csproj new file mode 100644 index 0000000..37c1eb9 --- /dev/null +++ b/tests/Prerender.AspNetCore.Tests.csproj @@ -0,0 +1,17 @@ + + + net8.0 + enable + enable + false + + + + + + + + + + + diff --git a/tests/PrerenderMiddlewareContractTests.cs b/tests/PrerenderMiddlewareContractTests.cs new file mode 100644 index 0000000..1b4f8cd --- /dev/null +++ b/tests/PrerenderMiddlewareContractTests.cs @@ -0,0 +1,217 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using System.Diagnostics; +using System.Net.Sockets; +using System.Text.Json; +using System.Text.RegularExpressions; +using Xunit; + +namespace Prerender.AspNetCore.Tests; + +// Contract tests against the shared mock server. +// Spec: https://github.com/prerender/integration-contract +// CI fetches mock-server.mjs into the repo root; locally: +// curl -fsSL -o mock-server.mjs https://raw.githubusercontent.com/prerender/integration-contract/main/mock-server.mjs + +public class MockServerFixture : IAsyncLifetime +{ + private const string DefaultMockPath = "mock-server.mjs"; + private Process? _process; + + public string Url { get; private set; } = string.Empty; + + public async Task InitializeAsync() + { + var mockPath = Environment.GetEnvironmentVariable("MOCK_SERVER_PATH") ?? DefaultMockPath; + var resolved = Path.IsPathRooted(mockPath) ? mockPath : Path.Combine(FindRepoRoot(), mockPath); + if (!File.Exists(resolved)) + { + throw new InvalidOperationException( + $"mock-server.mjs not found at {resolved}; fetch from prerender/integration-contract"); + } + + int port; + using (var listener = new TcpListener(System.Net.IPAddress.Loopback, 0)) + { + listener.Start(); + port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + } + + var psi = new ProcessStartInfo + { + FileName = "node", + ArgumentList = { resolved }, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + psi.Environment["PORT"] = port.ToString(); + _process = Process.Start(psi) ?? throw new InvalidOperationException("failed to start mock server"); + Url = $"http://127.0.0.1:{port}"; + + using var client = new HttpClient(); + for (var i = 0; i < 50; i++) + { + try + { + var r = await client.GetAsync($"{Url}/__health"); + if (r.IsSuccessStatusCode) return; + } + catch { /* not ready yet */ } + await Task.Delay(100); + } + throw new InvalidOperationException($"mock server at {Url} did not become ready"); + } + + public Task DisposeAsync() + { + try { _process?.Kill(true); } catch { /* ignore */ } + return Task.CompletedTask; + } + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "Prerender.AspNetCore.csproj"))) + dir = dir.Parent; + return dir?.FullName ?? Directory.GetCurrentDirectory(); + } +} + +public class PrerenderMiddlewareContractTests : IClassFixture +{ + private const string BotUserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1)"; + private const string BrowserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"; + private const string Token = "test-token-abc123"; + private static readonly Regex UuidV4 = new( + "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", + RegexOptions.IgnoreCase); + + private readonly MockServerFixture _mock; + private readonly HttpClient _bare = new(); + + public PrerenderMiddlewareContractTests(MockServerFixture mock) + { + _mock = mock; + } + + private async Task ResetAsync() => + await _bare.PostAsync($"{_mock.Url}/__reset", null); + + private async Task RecordedAsync() + { + var r = await _bare.GetAsync($"{_mock.Url}/__requests"); + var body = await r.Content.ReadAsStringAsync(); + return JsonDocument.Parse(body).RootElement; + } + + private TestServer CreateServer(string? token = Token) + { + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.Configure(opt => + { + opt.ServiceUrl = _mock.Url + "/"; + opt.Token = token; + }); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("original")); + }); + return new TestServer(builder); + } + + [Fact] + public async Task BotRequest_EmitsOutgoingRequestWithRequiredHeaders() + { + await ResetAsync(); + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + await client.GetAsync("/blog/post-1?ref=twitter"); + + var recorded = await RecordedAsync(); + Assert.Equal(1, recorded.GetArrayLength()); + var r = recorded[0]; + Assert.Equal("GET", r.GetProperty("method").GetString()); + Assert.EndsWith("/blog/post-1?ref=twitter", r.GetProperty("url").GetString()); + var headers = r.GetProperty("headers"); + Assert.Equal(BotUserAgent, headers.GetProperty("user-agent").GetString()); + Assert.Equal(Token, headers.GetProperty("x-prerender-token").GetString()); + Assert.Equal("AspNetCore", headers.GetProperty("x-prerender-int-type").GetString()); + Assert.Matches(@"^\d+\.\d+\.\d+", headers.GetProperty("x-prerender-int-version").GetString()!); + Assert.Matches(UuidV4, headers.GetProperty("x-prerender-request-id").GetString()!); + } + + [Fact] + public async Task BrowserRequest_EmitsNoOutgoingRequest() + { + await ResetAsync(); + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + await client.GetAsync("/"); + + var recorded = await RecordedAsync(); + Assert.Equal(0, recorded.GetArrayLength()); + } + + [Fact] + public async Task StaticAsset_EmitsNoOutgoingRequest() + { + await ResetAsync(); + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + await client.GetAsync("/styles.css"); + + var recorded = await RecordedAsync(); + Assert.Equal(0, recorded.GetArrayLength()); + } + + [Fact] + public async Task TokenOmitted_WhenUnconfigured() + { + await ResetAsync(); + using var server = CreateServer(token: null); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + await client.GetAsync("/"); + + var recorded = await RecordedAsync(); + Assert.Equal(1, recorded.GetArrayLength()); + Assert.False( + recorded[0].GetProperty("headers").TryGetProperty("x-prerender-token", out _), + "X-Prerender-Token must not be sent when unconfigured"); + } + + [Fact] + public async Task RequestId_IsUniquePerOutgoingRequest() + { + await ResetAsync(); + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + await client.GetAsync("/"); + await client.GetAsync("/"); + + var recorded = await RecordedAsync(); + Assert.Equal(2, recorded.GetArrayLength()); + Assert.NotEqual( + recorded[0].GetProperty("headers").GetProperty("x-prerender-request-id").GetString(), + recorded[1].GetProperty("headers").GetProperty("x-prerender-request-id").GetString()); + } +} diff --git a/tests/PrerenderMiddlewareTests.cs b/tests/PrerenderMiddlewareTests.cs new file mode 100644 index 0000000..df8ff58 --- /dev/null +++ b/tests/PrerenderMiddlewareTests.cs @@ -0,0 +1,166 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using System.Net; +using Xunit; + +namespace Prerender.AspNetCore.Tests; + +public class PrerenderMiddlewareTests +{ + private const string BotUserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1)"; + private const string BrowserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"; + private const string PrerenderedHtml = "prerendered"; + + private static TestServer CreateServer( + HttpResponseMessage? fakeResponse = null, + Action? configureOptions = null) + { + var response = fakeResponse ?? new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(PrerenderedHtml) + }; + + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FakeHttpMessageHandler(response)); + if (configureOptions is not null) + services.Configure(configureOptions); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + return new TestServer(builder); + } + + [Fact] + public async Task BrowserRequest_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_ReceivesPrerenderedResponse() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_StaticAsset_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/styles.css"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task EscapedFragment_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/?_escaped_fragment_="); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task XBufferbot_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + client.DefaultRequestHeaders.Add("X-Bufferbot", "true"); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task PostRequest_BotUa_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.PostAsync("/", new StringContent(string.Empty)); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task NetworkError_FallsBackToNormalResponse() + { + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FailingHttpMessageHandler()); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + using var server = new TestServer(builder); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } +} + +internal class FakeHttpMessageHandler : HttpMessageHandler +{ + private readonly HttpResponseMessage _response; + + public FakeHttpMessageHandler(HttpResponseMessage response) => _response = response; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(_response); +} + +internal class FailingHttpMessageHandler : HttpMessageHandler +{ + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("simulated network failure"); +}