diff --git a/apps/site/baseline/robots.txt b/apps/site/baseline/robots.txt index cc5da2d8..601c2456 100644 --- a/apps/site/baseline/robots.txt +++ b/apps/site/baseline/robots.txt @@ -1,6 +1,7 @@ User-Agent: * -Content-Signal: ai-train=no, search=yes, ai-input=yes +Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: / +Allow: /api/og/ Allow: /api/search Allow: /api/mcp Allow: /.well-known/ @@ -9,8 +10,26 @@ Allow: /llms-full.txt Allow: /auth.md Allow: /AGENTS.md Allow: /pricing.md -Disallow: /api/ Disallow: /dashboard/ +Disallow: /guest-portal/ + +User-Agent: Twitterbot +Allow: / + +User-Agent: Redditbot +Allow: / + +User-Agent: Discordbot +Allow: / + +User-Agent: Slackbot +Allow: / + +User-Agent: facebookexternalhit +Allow: / + +User-Agent: LinkedInBot +Allow: / User-Agent: GPTBot Allow: / diff --git a/apps/site/src/pages/robots.txt.ts b/apps/site/src/pages/robots.txt.ts index f161a269..72835f69 100644 --- a/apps/site/src/pages/robots.txt.ts +++ b/apps/site/src/pages/robots.txt.ts @@ -20,8 +20,9 @@ export const GET: APIRoute = async ({ locals }) => { const body = isProduction ? `User-Agent: * -Content-Signal: ai-train=no, search=yes, ai-input=yes +Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: / +Allow: /api/og/ Allow: /api/search Allow: /api/mcp Allow: /.well-known/ @@ -30,8 +31,26 @@ Allow: /llms-full.txt Allow: /auth.md Allow: /AGENTS.md Allow: /pricing.md -Disallow: /api/ Disallow: /dashboard/ +Disallow: /guest-portal/ + +User-Agent: Twitterbot +Allow: / + +User-Agent: Redditbot +Allow: / + +User-Agent: Discordbot +Allow: / + +User-Agent: Slackbot +Allow: / + +User-Agent: facebookexternalhit +Allow: / + +User-Agent: LinkedInBot +Allow: / User-Agent: GPTBot Allow: / diff --git a/apps/site/tests/tier1-features.test.mjs b/apps/site/tests/tier1-features.test.mjs index a04b5cd6..47ab84ec 100644 --- a/apps/site/tests/tier1-features.test.mjs +++ b/apps/site/tests/tier1-features.test.mjs @@ -653,15 +653,16 @@ describe("Tier 1 - Feature Coverage", () => { ); }); - it("4.2 Production robots allows root and disallows /api/ and /dashboard/", async () => { + it("4.2 Production robots allows root and /api/og/ and disallows /dashboard/ and /guest-portal/", async () => { const locals = { siteUrl: new URL("https://codingcat.dev") }; const res = await robotsModule.GET({ locals }); const body = await res.text(); assert.ok(body.includes("User-Agent: *")); assert.ok(body.includes("Allow: /")); - assert.ok(body.includes("Disallow: /api/")); + assert.ok(body.includes("Allow: /api/og/")); assert.ok(body.includes("Disallow: /dashboard/")); + assert.ok(body.includes("Disallow: /guest-portal/")); assert.ok(body.includes("Host: https://codingcat.dev")); assert.ok(body.includes("Sitemap: https://codingcat.dev/sitemap.xml")); }); diff --git a/apps/site/tests/tier4-baseline-contract.test.mjs b/apps/site/tests/tier4-baseline-contract.test.mjs index 69e0a7fa..2d98ab3e 100644 --- a/apps/site/tests/tier4-baseline-contract.test.mjs +++ b/apps/site/tests/tier4-baseline-contract.test.mjs @@ -320,10 +320,10 @@ describe("Tier 4 - Real-World Application & Baseline Contract", () => { const baselineRobots = loadBaseline("robots.txt"); const robotsModule = loadModule("src/pages/robots.txt.ts"); - it("4.1 Baseline robots contains User-Agent: * block with /api/ and /dashboard/ disallowed", () => { + it("4.1 Baseline robots contains User-Agent: * block with /api/og/ allowed and /dashboard/ disallowed", () => { assert.ok(baselineRobots.includes("User-Agent: *")); assert.ok(baselineRobots.includes("Allow: /")); - assert.ok(baselineRobots.includes("Disallow: /api/")); + assert.ok(baselineRobots.includes("Allow: /api/og/")); assert.ok(baselineRobots.includes("Disallow: /dashboard/")); });