sourcebot-dev · brendan-kellam · Aug 14, 2026 · Aug 14, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Fixed memory leak attributed to CodeMirror allocating objects on heap that were never freed. [#1580](https://github.com/sourcebot-dev/sourcebot/pull/1580)
 - Kept Git provider credentials out of subprocess arguments and on-disk configuration by using isolated in-memory credential caches. [#1584](https://github.com/sourcebot-dev/sourcebot/pull/1584)
+- Blocked `meta-externalagent` before server rendering when it ignores Sourcebot's crawler policy. [#1592](https://github.com/sourcebot-dev/sourcebot/pull/1592)
 
 ## [5.1.7] - 2026-08-13
 

diff --git a/packages/web/src/proxy.test.ts b/packages/web/src/proxy.test.ts
@@ -0,0 +1,37 @@
+import { NextRequest } from 'next/server';
+import { describe, expect, test } from 'vitest';
+import { proxy } from './proxy';
+
+describe('proxy', () => {
+    test.each([
+        'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
+        'Mozilla/5.0 (compatible; META-EXTERNALAGENT/1.1)',
+    ])('blocks the disallowed Meta crawler before rendering', async (userAgent) => {
+        const response = await proxy(new NextRequest('https://sourcebot.example/browse/github.com/sourcebot-dev/sourcebot', {
+            headers: {
+                'user-agent': userAgent,
+            },
+        }));
+
+        expect(response.status).toBe(403);
+        await expect(response.text()).resolves.toBe('Crawler access is disallowed.');
+    });
+
+    test('allows ordinary browser requests through', async () => {
+        const response = await proxy(new NextRequest('https://sourcebot.example/browse/github.com/sourcebot-dev/sourcebot', {
+            headers: {
+                'user-agent': 'Mozilla/5.0 Chrome/145.0.0.0',
+            },
+        }));
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('x-middleware-next')).toBe('1');
+    });
+
+    test('preserves the legacy organization-prefix redirect', async () => {
+        const response = await proxy(new NextRequest('https://sourcebot.example/~/search'));
+
+        expect(response.status).toBe(301);
+        expect(response.headers.get('location')).toBe('https://sourcebot.example/search');
+    });
+});
diff --git a/packages/web/src/proxy.ts b/packages/web/src/proxy.ts
@@ -2,6 +2,14 @@ import { StatusCodes } from 'http-status-codes';
 import { NextResponse } from 'next/server'
 import type { NextRequest } from 'next/server'
 
+// robots.txt disallows search and AI-training crawlers, but robots directives
+// are advisory. meta-externalagent has been observed ignoring that policy and
+// exhaustively walking the unbounded /browse URL space, so enforce the same
+// restriction before the request reaches server-side rendering.
+const BLOCKED_CRAWLER_USER_AGENTS = [
+    'meta-externalagent',
+] as const;
+
 /**
  * As part of our original SaaS effort in April 2025, we introduced
  * multi-tenancy into Sourcebot as part of v3.0.0. Organizations
@@ -24,6 +32,16 @@ import type { NextRequest } from 'next/server'
  * See: https://github.com/sourcebot-dev/sourcebot/pull/1076
  */
 export async function proxy(request: NextRequest) {
+    const userAgent = request.headers.get('user-agent')?.toLowerCase();
+    if (
+        userAgent
+        && BLOCKED_CRAWLER_USER_AGENTS.some(blockedCrawler => userAgent.includes(blockedCrawler))
+    ) {
+        return new NextResponse('Crawler access is disallowed.', {
+            status: StatusCodes.FORBIDDEN,
+        });
+    }
+
     const url = request.nextUrl.clone();
 
     if (url.pathname.startsWith('/~/')) {