seoBotMiddleware function
Middleware
seoBotMiddleware({
- List<
SeoRoute> ? routes, - SeoPageResolver? resolve,
- String? siteBase,
- bool serveSitemap = true,
- bool serveRobotsTxt = true,
- bool serveLlmsTxt = true,
- String? indexNowKey,
- List<
String> additionalSitemapPaths = const [], - bool unknownRoutesAs404 = true,
- BotDetector detector = const BotDetector(),
- SeoRedirectScope applyResolverRedirects = SeoRedirectScope.all,
- String? domFirstStylesheet = seoDefaultStylesheet,
- String? domFirstNonce(
- Request request
- SeoDomFirstRuntimeStore? domFirstRuntimeStore,
- Duration? infrastructureCacheTtl = seoAutoInfrastructureCacheTtl,
- void onResolveError(
- String path,
- Object error,
- StackTrace stack
Shelf middleware that serves semantic HTML to bots and passes real users through to the wrapped handler (usually the Flutter web build).
The recommended setup is the shared route table — one definition for app and server:
final handler = const Pipeline()
.addMiddleware(seoBotMiddleware(
routes: seoRoutes, // aus lib/seo_routes.dart
siteBase: 'https://esen.software',
))
.addHandler(createStaticHandler('build/web', defaultDocument: 'index.html'));
With routes set, the middleware additionally:
- derives canonical URLs from
siteBasefor routes without one, - serves
/sitemap.xml,/robots.txt,/llms.txtand/llms-full.txt(to every client, not just bots) generated from the table, - answers unknown page paths with a real HTTP 404 for bots
instead of the Flutter app — avoiding the classic SPA soft-404
problem. Paths whose last segment contains a dot (assets like
main.dart.js) always fall through to the wrapped handler.
With indexNowKey set, the IndexNow key file /<key>.txt is served
too, so submitIndexNow pings verify without extra hosting setup.
For special cases beyond the table, resolve is consulted when no
route matches.
Implementation
Middleware seoBotMiddleware({
List<SeoRoute>? routes,
SeoPageResolver? resolve,
String? siteBase,
bool serveSitemap = true,
bool serveRobotsTxt = true,
bool serveLlmsTxt = true,
String? indexNowKey,
List<String> additionalSitemapPaths = const [],
bool unknownRoutesAs404 = true,
BotDetector detector = const BotDetector(),
SeoRedirectScope applyResolverRedirects = SeoRedirectScope.all,
/// CSS embedded in every DOM-first document. Pass an empty string for an
/// intentionally unstyled page.
String? domFirstStylesheet = seoDefaultStylesheet,
/// Produces an optional CSP nonce for one DOM-first response.
///
/// The callback runs for the matched route only. It does not affect the
/// semantic content and is never consulted for Flutter-delivered routes.
String? Function(Request request)? domFirstNonce,
/// Resolves application-authored runtimes selected by DOM-first routes.
///
/// The middleware verifies the artifact on every delivery. Routes without an
/// application runtime never consult this store.
SeoDomFirstRuntimeStore? domFirstRuntimeStore,
Duration? infrastructureCacheTtl = seoAutoInfrastructureCacheTtl,
/// Called when a route resolver fails.
///
/// **Set this if you use dynamic routes.** The infrastructure
/// endpoints deliberately survive a failing page — a sitemap missing
/// one URL beats a sitemap that 500s — which means without this
/// callback that page disappears from `/sitemap.xml` and `/llms.txt`
/// with nothing to show for it. On the page path itself the error is
/// reported and then rethrown, so the crawler gets a 5xx and comes
/// back rather than indexing an empty shell.
void Function(String path, Object error, StackTrace stack)? onResolveError,
}) {
assert(
routes != null || resolve != null,
'seoBotMiddleware needs `routes` and/or `resolve`.',
);
final needsApplicationRuntime =
routes?.any((route) => route.applicationRuntime != null) ?? false;
if (needsApplicationRuntime && domFirstRuntimeStore == null) {
throw ArgumentError.notNull('domFirstRuntimeStore');
}
// Not just `isDynamic`: a CLASSIC route may carry an async
// `enumeratePaths` too, and the synchronous pass cannot await it
// either. Deciding on "is dynamic" alone made /sitemap.xml throw for a
// perfectly legal table.
final needsAsyncResolution =
routes?.any((r) => r.isDynamic || r.enumeratePaths != null) ?? false;
// Narrower than the above on purpose: only a `SeoRoute.dynamic` can
// ever resolve to a redirect. A classic route with an async
// `enumeratePaths` needs the async pass for the sitemap but can never
// produce one, so it must not pay for a head resolve on every human
// page view.
final hasDynamicRoute = routes?.any((r) => r.isDynamic) ?? false;
// "Classic" does not mean "immutable": a classic route's `body`
// builder may be async and read a database. Its output only reaches
// llms-full.txt (the sitemap and llms.txt need head detail, which
// never touches the body), but freezing THAT for the process lifetime
// would serve content captured at boot forever. It also has to take
// the async path so a failing body goes through the degrade-and-report
// policy instead of failing the whole endpoint.
final fullDetailMayDoIo = needsAsyncResolution ||
// ignore: deprecated_member_use_from_same_package
(routes?.any((r) => r.body != null) ?? false);
// Resolve the sentinel once, and separately per detail level: the
// sitemap and llms.txt only ever read head metadata, while
// llms-full.txt also renders bodies — so a table that is "static" for
// the first two can still be database-backed for the third.
const auto = seoAutoInfrastructureCacheTtl;
const autoTtl = Duration(minutes: 15);
final Duration? headTtl = infrastructureCacheTtl == auto
? (needsAsyncResolution ? autoTtl : null)
: infrastructureCacheTtl;
final Duration? fullTtl = infrastructureCacheTtl == auto
? (fullDetailMayDoIo ? autoTtl : null)
: infrastructureCacheTtl;
return (Handler inner) {
String? robotsCache; // pure siteBase, never changes — no TTL needed
void report(String p, Object e, StackTrace s) =>
onResolveError?.call(p, e, s);
// One cached Future per infrastructure file. A single shared Future
// per key does double duty: it caches the result until [infraTtl]
// expires (null = forever), and it collapses a stampede — twenty
// concurrent crawlers hitting /sitemap.xml trigger one pass, not
// twenty.
//
// Two things are deliberately NOT cached for the full TTL:
// * a build that throws is evicted at once, so the next request
// retries instead of replaying the error;
// * a build that DEGRADED — some rows failed and were dropped —
// is served this once but evicted too, so a transient database
// blip does not freeze an incomplete sitemap in place for
// 15 minutes after the database has recovered.
final cache = <String, Future<String>>{};
final timers = <String, Timer>{};
// Evict only if the slot still holds THIS generation. A build that
// outlives its own TTL would otherwise delete the newer entry a
// later request had already installed, quietly disabling the cache.
void evict(String key, Future<String> generation) {
if (!identical(cache[key], generation)) return;
cache.remove(key);
timers.remove(key)?.cancel();
}
Future<String> cached(
String key,
Duration? ttl,
Future<String> Function(void Function() markDegraded) build,
) {
final hit = cache[key];
if (hit != null) return hit;
var degraded = false;
late final Future<String> future;
future = () async {
try {
final result = await build(() => degraded = true);
if (degraded) evict(key, future);
return result;
} catch (_) {
evict(key, future);
rethrow;
}
}();
cache[key] = future;
if (ttl != null) {
timers[key] = Timer(ttl, () => evict(key, future));
}
return future;
}
return (Request request) async {
// SEO representations are read-only. A User-Agent is entirely
// caller-controlled, so letting it divert POST/PUT/DELETE would bypass
// the application's method-specific handler on the same path.
if (request.method != 'GET' && request.method != 'HEAD') {
return inner(request);
}
final path = _routePath(request.url.path, siteBase);
// Infrastruktur-Dateien — für alle Clients, nicht nur Bots.
if (siteBase != null) {
if (serveSitemap && routes != null && path == '/sitemap.xml') {
final xml = await cached(
'/sitemap.xml',
headTtl,
(markDegraded) async => needsAsyncResolution
? seoSitemapXml(
siteBase: siteBase,
pages: await resolveSeoPages(
routes: routes,
canonicalBase: siteBase,
additionalPaths: additionalSitemapPaths,
detail: SeoDetail.head,
onError: (p, e, st) {
markDegraded();
report(p, e, st);
},
),
)
: seoSitemapXml(
routes: routes,
siteBase: siteBase,
additionalPaths: additionalSitemapPaths,
),
);
return Response.ok(
xml,
headers: {'content-type': 'application/xml; charset=utf-8'},
);
}
if (serveRobotsTxt && path == '/robots.txt') {
robotsCache ??=
seoRobotsTxt(siteBase: siteBase, includeSitemap: serveSitemap);
return Response.ok(
robotsCache,
headers: {'content-type': 'text/plain; charset=utf-8'},
);
}
if (serveLlmsTxt && routes != null && path == '/llms.txt') {
final txt = await cached(
'/llms.txt',
headTtl,
(markDegraded) async => needsAsyncResolution
? seoLlmsTxt(
siteBase: siteBase,
pages: await resolveSeoPages(
routes: routes,
canonicalBase: siteBase,
additionalPaths: additionalSitemapPaths,
detail: SeoDetail.head,
onError: (p, e, st) {
markDegraded();
report(p, e, st);
},
),
)
: seoLlmsTxt(
routes: routes,
siteBase: siteBase,
additionalPaths: additionalSitemapPaths,
),
);
return Response.ok(
txt,
headers: {'content-type': 'text/plain; charset=utf-8'},
);
}
if (serveLlmsTxt && routes != null && path == '/llms-full.txt') {
final txt = await cached(
'/llms-full.txt',
fullTtl,
(markDegraded) async => fullDetailMayDoIo
? seoLlmsFullTxt(
siteBase: siteBase,
pages: await resolveSeoPages(
routes: routes,
canonicalBase: siteBase,
additionalPaths: additionalSitemapPaths,
detail: SeoDetail.full,
onError: (p, e, st) {
markDegraded();
report(p, e, st);
},
),
)
: seoLlmsFullTxt(
routes: routes,
siteBase: siteBase,
additionalPaths: additionalSitemapPaths,
),
);
return Response.ok(
txt,
headers: {'content-type': 'text/plain; charset=utf-8'},
);
}
}
if (indexNowKey != null && path == '/$indexNowKey.txt') {
return Response.ok(
indexNowKey,
headers: {'content-type': 'text/plain; charset=utf-8'},
);
}
// DOM-first is a route property, not a User-Agent representation. It
// resolves before bot detection and every outcome is final because
// there is no Flutter application on this route to fall back to.
if (routes != null) {
final match = matchSeoRoute(routes, path);
if (match != null && match.route.isDomFirst) {
final SeoResolution resolution;
try {
resolution = await match.resolve(
canonicalBase: siteBase,
onWarning: (p, w) => report(p, StateError(w), StackTrace.current),
);
} catch (error, stack) {
report(path, error, stack);
rethrow;
}
switch (resolution) {
case SeoRedirect(:final location, :final statusCode):
return Response(statusCode, headers: {'location': location});
case SeoDocument(
:final statusCode,
:final body,
:final meta,
:final headers,
):
final runtimeReference = match.route.applicationRuntime;
final applicationRuntime = runtimeReference == null
? null
: await loadSeoDomFirstRuntime(
domFirstRuntimeStore!,
runtimeReference,
);
final pageBody = statusCode >= 400 && body.isEmpty
? _statusBody(statusCode)
: body;
return _htmlResponse(
SeoPage.domFirstFromNodes(
meta: statusCode >= 400 && meta.title == null
? meta.copyWith(title: _statusTitle(statusCode))
: meta,
body: pageBody,
lang: resolution.lang ?? match.route.lang,
stylesheet: domFirstStylesheet,
features: match.route.domFirstFeatures,
applicationRuntime: applicationRuntime,
interactionNonce: domFirstNonce?.call(request),
),
status: statusCode,
extraHeaders: _safeHeaders(headers, varyUserAgent: false),
varyUserAgent: false,
surface: 'dom-first',
);
}
}
}
final isBot = detector.isBot(request.headers['user-agent']);
// A resolver redirect applies to humans too by default — a 301
// shown only to Googlebot is cloaking, so bots and users must
// reach the same destination. Only a dynamic table can produce a
// redirect (a static resolution never is one), and only when the
// scope allows it. A resolver failure here must never 5xx a human:
// it is reported and falls through to the app, which still renders.
// No `_looksLikePage` filter here, deliberately: the bot branch
// has none either, and gating only this side made a redirect for a
// dotted path (`/old-page.html` → clean URL, the commonest
// relaunch mapping there is) reach crawlers but not humans — the
// exact cloaking this mode exists to prevent. `matchSeoRoute` is
// already the right filter: an asset request matches no route and
// costs nothing.
if (!isBot &&
hasDynamicRoute &&
routes != null &&
applyResolverRedirects == SeoRedirectScope.all) {
final match = matchSeoRoute(routes, path);
// Per route, not per table: in a mixed table the classic routes
// must not pay a meta build on every human page view just
// because a dynamic route exists somewhere else. Only a dynamic
// route can resolve to a redirect.
if (match != null && match.route.isDynamic) {
try {
final res = await match.resolve(
detail: SeoDetail.head,
canonicalBase: siteBase,
onWarning: (p, w) => report(p, StateError(w), StackTrace.current),
);
if (res is SeoRedirect) {
return Response(
res.statusCode,
headers: {'location': res.location, ..._varyHeader},
);
}
} catch (error, stack) {
report(path, error, stack);
}
}
}
if (!isBot) {
return _appResponse(inner, request);
}
var routeExists = false;
if (routes != null) {
final match = matchSeoRoute(routes, path);
if (match != null) {
routeExists = true;
final SeoResolution resolution;
try {
resolution = await match.resolve(
canonicalBase: siteBase,
onWarning: (p, w) => report(p, StateError(w), StackTrace.current),
);
} catch (error, stack) {
// The page path used to be the one place a resolver failure
// bypassed onResolveError entirely. Report it, then let it
// through: a 5xx tells a crawler to come back, while serving
// the empty Flutter shell with a 200 invites it to index
// nothing at all.
report(path, error, stack);
rethrow;
}
switch (resolution) {
case SeoRedirect(:final location, :final statusCode):
// Under `off` the redirect is ignored and the request
// falls through to the app (pair with seoRedirectMiddleware
// to drive 301s from a map instead). Otherwise the bot gets
// it; the human branch above already handled `all`.
if (applyResolverRedirects == SeoRedirectScope.off) break;
return Response(
statusCode,
headers: {'location': location, ..._varyHeader},
);
case SeoDocument(:final statusCode, :final body, :final meta)
when statusCode >= 400:
// A real error status with the document's own body, or the
// built-in error markup when it carries none — but always
// with the resolver's own metadata. `SeoDocument.notFound`
// takes a `meta:` argument, so silently swapping in a
// generic page would throw away a title and description
// the caller deliberately supplied.
return _htmlResponse(
SeoPage.fromNodes(
meta: meta.title == null
? meta.copyWith(title: _statusTitle(statusCode))
: meta,
body: body.isEmpty ? _statusBody(statusCode) : body,
lang: resolution.lang ?? match.route.lang,
),
status: statusCode,
extraHeaders: _safeHeaders(resolution.headers),
);
case SeoDocument(:final body, :final meta):
// A 200 with nothing to mirror falls through to the app —
// an empty SSR page indexes worse than the Flutter shell.
if (body.isNotEmpty) {
return _htmlResponse(
SeoPage.fromNodes(
meta: meta,
body: body,
lang: resolution.lang ?? match.route.lang,
),
extraHeaders: _safeHeaders(resolution.headers),
);
}
}
}
}
if (resolve != null) {
final page = await resolve(request);
if (page != null) return _htmlResponse(page);
}
// Soft-404 vermeiden: unbekannte seiten-artige Pfade bekommen für
// Bots einen echten 404 statt der Flutter-App mit Status 200.
// Eine Route, die es GIBT, gehört niemals hierher — sie hat nur
// keinen Body, und dann ist die App die richtige Antwort.
if (routes != null &&
!routeExists &&
unknownRoutesAs404 &&
looksLikeSeoPagePath(path)) {
return _htmlResponse(_notFoundPage(), status: 404);
}
return _appResponse(inner, request);
};
};
}