diff --git a/prerender-aspnetcore/Prerender.AspNetCore.csproj b/prerender-aspnetcore/Prerender.AspNetCore.csproj new file mode 100644 index 0000000..41af1fd --- /dev/null +++ b/prerender-aspnetcore/Prerender.AspNetCore.csproj @@ -0,0 +1,21 @@ + + + net8.0 + enable + enable + Prerender.AspNetCore + 1.0.0 + Prerender.io + ASP.NET Core middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io + MIT + README.md + https://github.com/prerender/integrations + git + + + + + + + + diff --git a/prerender-aspnetcore/PrerenderMiddleware.cs b/prerender-aspnetcore/PrerenderMiddleware.cs new file mode 100644 index 0000000..c9f05b5 --- /dev/null +++ b/prerender-aspnetcore/PrerenderMiddleware.cs @@ -0,0 +1,106 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Prerender.AspNetCore; + +public class PrerenderMiddleware : IMiddleware +{ + private static readonly string[] CrawlerUserAgents = + [ + "googlebot", "yahoo", "bingbot", "baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", + "embedly", "quora link preview", "showyoubot", "outbrain", + "pinterest", "slackbot", "w3c_validator", "perplexity", + "oai-searchbot", "chatgpt-user", "gptbot", "claudebot", "amazonbot", + ]; + + private static readonly string[] ExtensionsToIgnore = + [ + ".js", ".css", ".xml", ".less", ".png", ".jpg", ".jpeg", ".gif", + ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", + ".exe", ".wmv", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", + ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", + ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".ttf", ".woff", ".svg", + ]; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly PrerenderOptions _options; + private readonly ILogger _logger; + + public PrerenderMiddleware( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _options = options.Value; + _logger = logger; + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + if (!ShouldPrerender(context)) + { + await next(context); + return; + } + + try + { + var client = _httpClientFactory.CreateClient("prerender"); + var apiUrl = BuildApiUrl(context); + + using var request = new HttpRequestMessage(HttpMethod.Get, apiUrl); + request.Headers.TryAddWithoutValidation( + "User-Agent", context.Request.Headers["User-Agent"].ToString()); + if (!string.IsNullOrWhiteSpace(_options.Token)) + request.Headers.TryAddWithoutValidation("X-Prerender-Token", _options.Token); + + using var response = await client.SendAsync(request, context.RequestAborted); + context.Response.StatusCode = (int)response.StatusCode; + var body = await response.Content.ReadAsStringAsync(); + await context.Response.WriteAsync(body); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "Prerender service unreachable, falling back"); + await next(context); + } + } + + private static bool ShouldPrerender(HttpContext context) + { + if (context.Request.Method != HttpMethods.Get) return false; + + var path = context.Request.Path.Value ?? string.Empty; + if (IsStaticAsset(path)) return false; + + if (context.Request.Query.ContainsKey("_escaped_fragment_")) return true; + if (context.Request.Headers.ContainsKey("X-Bufferbot")) return true; + + var ua = context.Request.Headers["User-Agent"].ToString(); + return !string.IsNullOrEmpty(ua) && IsBot(ua); + } + + private string BuildApiUrl(HttpContext context) + { + var serviceUrl = _options.ServiceUrl.TrimEnd('/') + "/"; + var scheme = context.Request.Scheme; + var host = context.Request.Host.Value; + var pathAndQuery = context.Request.Path + context.Request.QueryString; + return $"{serviceUrl}{scheme}://{host}{pathAndQuery}"; + } + + private static bool IsBot(string userAgent) + { + var ua = userAgent.ToLowerInvariant(); + return CrawlerUserAgents.Any(bot => ua.Contains(bot)); + } + + private static bool IsStaticAsset(string path) + { + var lower = path.ToLowerInvariant(); + return ExtensionsToIgnore.Any(ext => lower.EndsWith(ext)); + } +} diff --git a/prerender-aspnetcore/PrerenderOptions.cs b/prerender-aspnetcore/PrerenderOptions.cs new file mode 100644 index 0000000..715f494 --- /dev/null +++ b/prerender-aspnetcore/PrerenderOptions.cs @@ -0,0 +1,7 @@ +namespace Prerender.AspNetCore; + +public class PrerenderOptions +{ + public string? Token { get; set; } + public string ServiceUrl { get; set; } = "https://service.prerender.io/"; +} diff --git a/prerender-aspnetcore/PrerenderServiceExtensions.cs b/prerender-aspnetcore/PrerenderServiceExtensions.cs new file mode 100644 index 0000000..30ac9d8 --- /dev/null +++ b/prerender-aspnetcore/PrerenderServiceExtensions.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Prerender.AspNetCore; + +public static class PrerenderServiceExtensions +{ + public static IServiceCollection AddPrerender(this IServiceCollection services) + { + services.AddOptions().BindConfiguration("Prerender"); + services.AddHttpClient("prerender"); + services.AddTransient(); + return services; + } + + public static IApplicationBuilder UsePrerender(this IApplicationBuilder app) + => app.UseMiddleware(); +} diff --git a/prerender-aspnetcore/README.md b/prerender-aspnetcore/README.md new file mode 100644 index 0000000..b5b6aac --- /dev/null +++ b/prerender-aspnetcore/README.md @@ -0,0 +1,69 @@ +# prerender-aspnetcore + +ASP.NET Core middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers. + +Compatible with **ASP.NET Core 8+** and **.NET 8+**. + +## Installation + +```bash +dotnet add package Prerender.AspNetCore +``` + +## Setup + +Register the middleware in `Program.cs`: + +```csharp +builder.Services.AddPrerender(); + +var app = builder.Build(); +app.UsePrerender(); // place before routing middleware +``` + +Add your token to `appsettings.json`: + +```json +{ + "Prerender": { + "Token": "YOUR_PRERENDER_TOKEN" + } +} +``` + +The middleware must be placed **before** routing to intercept bot requests early. + +## Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| `Prerender:Token` | `null` | Your Prerender.io token | +| `Prerender:ServiceUrl` | `https://service.prerender.io/` | Prerender service URL (override for self-hosted Prerender) | + +## Self-hosted Prerender + +```json +{ + "Prerender": { + "ServiceUrl": "http://your-prerender-server:3000" + } +} +``` + +## How it works + +Requests are prerendered when **all** of the following are true: + +- The HTTP method is `GET` +- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.) + — OR the URL contains `_escaped_fragment_` + — OR the `X-Bufferbot` header is present +- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.) + +Everything else passes through to your normal ASP.NET Core pipeline. + +If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response. + +## License + +MIT diff --git a/prerender-aspnetcore/tests/Prerender.AspNetCore.Tests.csproj b/prerender-aspnetcore/tests/Prerender.AspNetCore.Tests.csproj new file mode 100644 index 0000000..37c1eb9 --- /dev/null +++ b/prerender-aspnetcore/tests/Prerender.AspNetCore.Tests.csproj @@ -0,0 +1,17 @@ + + + net8.0 + enable + enable + false + + + + + + + + + + + diff --git a/prerender-aspnetcore/tests/PrerenderMiddlewareTests.cs b/prerender-aspnetcore/tests/PrerenderMiddlewareTests.cs new file mode 100644 index 0000000..df8ff58 --- /dev/null +++ b/prerender-aspnetcore/tests/PrerenderMiddlewareTests.cs @@ -0,0 +1,166 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using System.Net; +using Xunit; + +namespace Prerender.AspNetCore.Tests; + +public class PrerenderMiddlewareTests +{ + private const string BotUserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1)"; + private const string BrowserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"; + private const string PrerenderedHtml = "prerendered"; + + private static TestServer CreateServer( + HttpResponseMessage? fakeResponse = null, + Action? configureOptions = null) + { + var response = fakeResponse ?? new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(PrerenderedHtml) + }; + + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FakeHttpMessageHandler(response)); + if (configureOptions is not null) + services.Configure(configureOptions); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + return new TestServer(builder); + } + + [Fact] + public async Task BrowserRequest_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_ReceivesPrerenderedResponse() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_StaticAsset_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/styles.css"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task EscapedFragment_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/?_escaped_fragment_="); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task XBufferbot_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + client.DefaultRequestHeaders.Add("X-Bufferbot", "true"); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task PostRequest_BotUa_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.PostAsync("/", new StringContent(string.Empty)); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task NetworkError_FallsBackToNormalResponse() + { + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FailingHttpMessageHandler()); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + using var server = new TestServer(builder); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } +} + +internal class FakeHttpMessageHandler : HttpMessageHandler +{ + private readonly HttpResponseMessage _response; + + public FakeHttpMessageHandler(HttpResponseMessage response) => _response = response; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(_response); +} + +internal class FailingHttpMessageHandler : HttpMessageHandler +{ + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("simulated network failure"); +} diff --git a/prerender-django/.env.example b/prerender-django/.env.example new file mode 100644 index 0000000..07bced2 --- /dev/null +++ b/prerender-django/.env.example @@ -0,0 +1,2 @@ +PRERENDER_TOKEN= +PRERENDER_SERVICE_URL= diff --git a/prerender-django/.gitignore b/prerender-django/.gitignore new file mode 100644 index 0000000..3c9f148 --- /dev/null +++ b/prerender-django/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.env diff --git a/prerender-django/README.md b/prerender-django/README.md new file mode 100644 index 0000000..9d4d1ea --- /dev/null +++ b/prerender-django/README.md @@ -0,0 +1,57 @@ +# prerender-django + +Django middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers. + +Compatible with **Django 5+** and **Python 3.10+**. + +## Installation + +```bash +pip install prerender-django +``` + +## Setup + +Add the middleware to your `settings.py`: + +```python +MIDDLEWARE = [ + 'prerender_django.middleware.PrerenderMiddleware', + # ... your other middleware +] + +PRERENDER_TOKEN = 'YOUR_PRERENDER_TOKEN' +``` + +The middleware must be placed **before** any session or authentication middleware to intercept bot requests early. + +## Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| `PRERENDER_TOKEN` | `None` | Your Prerender.io token | +| `PRERENDER_SERVICE_URL` | `https://service.prerender.io/` | Prerender service URL (use this for self-hosted Prerender) | + +## Self-hosted Prerender + +```python +PRERENDER_SERVICE_URL = 'http://your-prerender-server:3000' +``` + +## How it works + +Requests are prerendered when **all** of the following are true: + +- The HTTP method is `GET` +- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.) + — OR the URL contains `_escaped_fragment_` + — OR the `X-Bufferbot` header is present +- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.) + +Everything else passes through to your normal Django views. + +If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response. + +## License + +MIT diff --git a/prerender-django/prerender_django/__init__.py b/prerender-django/prerender_django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prerender-django/prerender_django/middleware.py b/prerender-django/prerender_django/middleware.py new file mode 100644 index 0000000..a982f1c --- /dev/null +++ b/prerender-django/prerender_django/middleware.py @@ -0,0 +1,89 @@ +import logging +import urllib.error +import urllib.request + +from django.conf import settings +from django.http import HttpResponse + +logger = logging.getLogger(__name__) + +CRAWLER_USER_AGENTS = [ + 'googlebot', 'yahoo', 'bingbot', 'baiduspider', + 'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot', + 'embedly', 'quora link preview', 'showyoubot', 'outbrain', + 'pinterest', 'slackbot', 'developers.google.com/+/web/snippet', + 'w3c_validator', 'perplexity', 'oai-searchbot', 'chatgpt-user', + 'gptbot', 'claudebot', 'amazonbot', +] + +EXTENSIONS_TO_IGNORE = frozenset([ + '.js', '.css', '.xml', '.less', '.png', '.jpg', '.jpeg', '.gif', + '.pdf', '.doc', '.txt', '.ico', '.rss', '.zip', '.mp3', '.rar', + '.exe', '.wmv', '.avi', '.ppt', '.mpg', '.mpeg', '.tif', '.wav', + '.mov', '.psd', '.ai', '.xls', '.mp4', '.m4a', '.swf', '.dat', + '.dmg', '.iso', '.flv', '.m4v', '.torrent', '.ttf', '.woff', '.svg', +]) + + +def _setting(name, default=None): + return getattr(settings, f'PRERENDER_{name}', default) + + +def _is_bot(user_agent): + ua = user_agent.lower() + return any(bot in ua for bot in CRAWLER_USER_AGENTS) + + +def _is_static_asset(path): + return any(path.endswith(ext) for ext in EXTENSIONS_TO_IGNORE) + + +def _should_prerender(request): + user_agent = request.META.get('HTTP_USER_AGENT', '') + if not user_agent or request.method != 'GET': + return False + if _is_static_asset(request.path): + return False + if '_escaped_fragment_' in request.GET: + return True + if request.META.get('HTTP_X_BUFFERBOT'): + return True + return _is_bot(user_agent) + + +def _build_api_url(request): + service_url = _setting('SERVICE_URL', 'https://service.prerender.io/') + if not service_url.endswith('/'): + service_url += '/' + return f'{service_url}{request.build_absolute_uri()}' + + +def _fetch_prerendered(api_url, user_agent): + token = _setting('TOKEN') + req = urllib.request.Request(api_url) + req.add_header('User-Agent', user_agent) + if token: + req.add_header('X-Prerender-Token', token) + try: + with urllib.request.urlopen(req) as resp: + return resp.status, resp.read().decode('utf-8') + except urllib.error.HTTPError as e: + return e.code, e.read().decode('utf-8') + + +class PrerenderMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if not _should_prerender(request): + return self.get_response(request) + + try: + api_url = _build_api_url(request) + user_agent = request.META.get('HTTP_USER_AGENT', '') + status, body = _fetch_prerendered(api_url, user_agent) + return HttpResponse(body, status=status, content_type='text/html') + except urllib.error.URLError as e: + logger.error('Prerender error, falling back: %s', e) + return self.get_response(request) diff --git a/prerender-django/pyproject.toml b/prerender-django/pyproject.toml new file mode 100644 index 0000000..30a979d --- /dev/null +++ b/prerender-django/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "prerender-django" +version = "1.0.1" +description = "Django middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io" +authors = [{ name = "Prerender.io" }] +license = "MIT" +readme = "README.md" +requires-python = ">=3.10" +keywords = ["django", "prerender", "prerender.io", "seo", "middleware"] +dependencies = [] + +[project.urls] +Repository = "https://github.com/prerender/integrations" + +[project.optional-dependencies] +dev = [ + "django>=5.0", + "pytest>=8.0", + "pytest-django>=4.8", +] + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "tests.settings" +pythonpath = ["."] + +[tool.setuptools.packages.find] +include = ["prerender_django*"] diff --git a/prerender-django/tests/__init__.py b/prerender-django/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prerender-django/tests/settings.py b/prerender-django/tests/settings.py new file mode 100644 index 0000000..07608ba --- /dev/null +++ b/prerender-django/tests/settings.py @@ -0,0 +1,3 @@ +SECRET_KEY = 'test-secret-key' +DATABASES = {} +INSTALLED_APPS = [] diff --git a/prerender-django/tests/test_middleware.py b/prerender-django/tests/test_middleware.py new file mode 100644 index 0000000..d78b9f5 --- /dev/null +++ b/prerender-django/tests/test_middleware.py @@ -0,0 +1,69 @@ +import urllib.error +from unittest.mock import MagicMock, patch + +from django.http import HttpResponse +from django.test import RequestFactory + +from prerender_django.middleware import PrerenderMiddleware + +BOT_UA = 'Mozilla/5.0 (compatible; Googlebot/2.1)' +BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' +PRERENDERED_HTML = 'prerendered' + +factory = RequestFactory() + + +def normal_response(_request): + return HttpResponse('original') + + +def mock_urlopen(status=200, body=PRERENDERED_HTML): + cm = MagicMock() + cm.__enter__ = MagicMock(return_value=cm) + cm.__exit__ = MagicMock(return_value=False) + cm.status = status + cm.read.return_value = body.encode('utf-8') + return cm + + +def test_browser_passes_through(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BROWSER_UA) + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original' + + +def test_bot_receives_prerendered_response(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BOT_UA) + with patch('urllib.request.urlopen', return_value=mock_urlopen()): + response = middleware(request) + assert response.status_code == 200 + assert PRERENDERED_HTML in response.content.decode() + + +def test_static_asset_with_bot_ua_passes_through(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/style.css', HTTP_USER_AGENT=BOT_UA) + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original' + + +def test_escaped_fragment_triggers_prerender(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', {'_escaped_fragment_': ''}, HTTP_USER_AGENT=BROWSER_UA) + with patch('urllib.request.urlopen', return_value=mock_urlopen()): + response = middleware(request) + assert response.status_code == 200 + assert PRERENDERED_HTML in response.content.decode() + + +def test_network_error_falls_back_to_normal_response(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BOT_UA) + with patch('urllib.request.urlopen', side_effect=urllib.error.URLError('network error')): + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original'