improvement(seo): optimize sitemaps, robots.txt, and core web vitals …

…across sim and docs (#4170) * improvement(seo): optimize sitemaps and robots.txt across sim and docs - Add missing pages to sim sitemap: blog author pages, academy catalog and course pages - Fix 6x duplicate URL bug in docs sitemap by deduplicating with source.getLanguages() - Convert docs sitemap from route handler to Next.js metadata convention with native hreflang - Add x-default hreflang alternate for docs multi-language pages - Remove changeFrequency and priority fields (Google ignores both) - Fix inaccurate lastModified timestamps — derive from real content dates, omit when unknown - Consolidate 20+ redundant per-bot robots rules into single wildcard entry - Add /form/ and /credential-account/ to sim robots disallow list - Reference image sitemap in sim robots.txt - Remove deprecated host directive from sim robots - Move disallow rules before allow in docs robots for crawler compatibility - Extract hardcoded docs baseUrl to env variable with production fallback * fix(seo): remove homepage new Date(), guard latestModelDate empty array * improvement(seo): consolidate DOCS_BASE_URL, optimize core web vitals Extract hardcoded https://docs.sim.ai into shared DOCS_BASE_URL constant in lib/urls.ts and replace all 20+ instances across layouts, metadata, structured data, LLM manifest, sitemap, and robots files. Remove OneDollarStats analytics script and tighten CSP for improved core web vitals. * fix: removed onedollarstats from bun lock * fix(seo): guard per-provider Math.max, consolidate docs robots to single wildcard
simstudioai · Sg312 · Apr 15, 2026 · Apr 15, 2026 · Apr 15, 2026 · Apr 15, 2026
commit 5274efd8f97b7254755b19d6c6fc8c908bcc3662
diff --git a/apps/docs/app/[lang]/[[...slug]]/page.tsx b/apps/docs/app/[lang]/[[...slug]]/page.tsx
@@ -17,9 +17,10 @@ import { ResponseSection } from '@/components/ui/response-section'
 import { i18n } from '@/lib/i18n'
 import { getApiSpecContent, openapi } from '@/lib/openapi'
 import { type PageData, source } from '@/lib/source'
+import { DOCS_BASE_URL } from '@/lib/urls'
 
 const SUPPORTED_LANGUAGES: Set<string> = new Set(i18n.languages)
-const BASE_URL = 'https://docs.sim.ai'
+const BASE_URL = DOCS_BASE_URL
 
 const OG_LOCALE_MAP: Record<string, string> = {
   en: 'en_US',

diff --git a/apps/docs/app/[lang]/layout.tsx b/apps/docs/app/[lang]/layout.tsx
@@ -3,7 +3,6 @@ import { defineI18nUI } from 'fumadocs-ui/i18n'
 import { DocsLayout } from 'fumadocs-ui/layouts/docs'
 import { RootProvider } from 'fumadocs-ui/provider/next'
 import { Geist_Mono, Inter } from 'next/font/google'
-import Script from 'next/script'
 import {
   SidebarFolder,
   SidebarItem,
@@ -13,6 +12,7 @@ import { Navbar } from '@/components/navbar/navbar'
 import { SimLogoFull } from '@/components/ui/sim-logo'
 import { i18n } from '@/lib/i18n'
 import { source } from '@/lib/source'
+import { DOCS_BASE_URL } from '@/lib/urls'
 import '../global.css'
 
 const inter = Inter({
@@ -67,22 +67,22 @@ export default async function Layout({ children, params }: LayoutProps) {
     name: 'Sim Documentation',
     description:
       'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
-    url: 'https://docs.sim.ai',
+    url: DOCS_BASE_URL,
     publisher: {
       '@type': 'Organization',
       name: 'Sim',
       url: 'https://sim.ai',
       logo: {
         '@type': 'ImageObject',
-        url: 'https://docs.sim.ai/static/logo.png',
+        url: `${DOCS_BASE_URL}/static/logo.png`,
       },
     },
     inLanguage: lang,
     potentialAction: {
       '@type': 'SearchAction',
       target: {
         '@type': 'EntryPoint',
-        urlTemplate: 'https://docs.sim.ai/api/search?q={search_term_string}',
+        urlTemplate: `${DOCS_BASE_URL}/api/search?q={search_term_string}`,
       },
       'query-input': 'required name=search_term_string',
     },
@@ -101,7 +101,6 @@ export default async function Layout({ children, params }: LayoutProps) {
         />
       </head>
       <body className='flex min-h-screen flex-col font-sans'>
-        <Script src='https://assets.onedollarstats.com/stonks.js' strategy='lazyOnload' />
         <RootProvider i18n={provider(lang)}>
           <Navbar />
           <DocsLayout

diff --git a/apps/docs/app/layout.tsx b/apps/docs/app/layout.tsx
@@ -1,5 +1,6 @@
 import type { ReactNode } from 'react'
 import type { Viewport } from 'next'
+import { DOCS_BASE_URL } from '@/lib/urls'
 
 export default function RootLayout({ children }: { children: ReactNode }) {
   return children
@@ -12,7 +13,7 @@ export const viewport: Viewport = {
 }
 
 export const metadata = {
-  metadataBase: new URL('https://docs.sim.ai'),
+  metadataBase: new URL(DOCS_BASE_URL),
   title: {
     default: 'Sim Documentation — The AI Workspace for Teams',
     template: '%s | Sim Docs',
@@ -61,14 +62,14 @@ export const metadata = {
     type: 'website',
     locale: 'en_US',
     alternateLocale: ['es_ES', 'fr_FR', 'de_DE', 'ja_JP', 'zh_CN'],
-    url: 'https://docs.sim.ai',
+    url: DOCS_BASE_URL,
     siteName: 'Sim Documentation',
     title: 'Sim Documentation — The AI Workspace for Teams',
     description:
       'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
     images: [
       {
-        url: 'https://docs.sim.ai/api/og?title=Sim%20Documentation',
+        url: `${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`,
         width: 1200,
         height: 630,
         alt: 'Sim Documentation',
@@ -82,7 +83,7 @@ export const metadata = {
       'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
     creator: '@simdotai',
     site: '@simdotai',
-    images: ['https://docs.sim.ai/api/og?title=Sim%20Documentation'],
+    images: [`${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`],
   },
   robots: {
     index: true,
@@ -96,15 +97,15 @@ export const metadata = {
     },
   },
   alternates: {
-    canonical: 'https://docs.sim.ai',
+    canonical: DOCS_BASE_URL,
     languages: {
-      'x-default': 'https://docs.sim.ai',
-      en: 'https://docs.sim.ai',
-      es: 'https://docs.sim.ai/es',
-      fr: 'https://docs.sim.ai/fr',
-      de: 'https://docs.sim.ai/de',
-      ja: 'https://docs.sim.ai/ja',
-      zh: 'https://docs.sim.ai/zh',
+      'x-default': DOCS_BASE_URL,
+      en: DOCS_BASE_URL,
+      es: `${DOCS_BASE_URL}/es`,
+      fr: `${DOCS_BASE_URL}/fr`,
+      de: `${DOCS_BASE_URL}/de`,
+      ja: `${DOCS_BASE_URL}/ja`,
+      zh: `${DOCS_BASE_URL}/zh`,
     },
   },
 }
diff --git a/apps/docs/app/llms.txt/route.ts b/apps/docs/app/llms.txt/route.ts
@@ -1,9 +1,10 @@
 import { source } from '@/lib/source'
+import { DOCS_BASE_URL } from '@/lib/urls'
 
 export const revalidate = false
 
 export async function GET() {
-  const baseUrl = 'https://docs.sim.ai'
+  const baseUrl = DOCS_BASE_URL
 
   try {
     const pages = source.getPages().filter((page) => {

diff --git a/apps/docs/app/robots.txt/route.ts b/apps/docs/app/robots.txt/route.ts
@@ -1,70 +1,18 @@
+import { DOCS_BASE_URL } from '@/lib/urls'
+
 export const revalidate = false
 
 export async function GET() {
-  const baseUrl = 'https://docs.sim.ai'
+  const baseUrl = DOCS_BASE_URL
 
   const robotsTxt = `# Robots.txt for Sim Documentation
 
 User-agent: *
-Allow: /
-
-# Search engine crawlers
-User-agent: Googlebot
-Allow: /
-
-User-agent: Bingbot
-Allow: /
-
-User-agent: Slurp
-Allow: /
-
-User-agent: DuckDuckBot
-Allow: /
-
-User-agent: Baiduspider
-Allow: /
-
-User-agent: YandexBot
-Allow: /
-
-# AI and LLM crawlers - explicitly allowed for documentation indexing
-User-agent: GPTBot
-Allow: /
-
-User-agent: ChatGPT-User
-Allow: /
-
-User-agent: CCBot
-Allow: /
-
-User-agent: anthropic-ai
-Allow: /
-
-User-agent: Claude-Web
-Allow: /
-
-User-agent: Applebot
-Allow: /
-
-User-agent: PerplexityBot
-Allow: /
-
-User-agent: Diffbot
-Allow: /
-
-User-agent: FacebookBot
-Allow: /
-
-User-agent: cohere-ai
-Allow: /
-
-# Disallow admin and internal paths (if any exist)
 Disallow: /.next/
 Disallow: /api/internal/
 Disallow: /_next/static/
 Disallow: /admin/
-
-# Allow but don't prioritize these
+Allow: /
 Allow: /api/search
 Allow: /llms.txt
 Allow: /llms-full.txt
@@ -73,23 +21,12 @@ Allow: /llms.mdx/
 # Sitemaps
 Sitemap: ${baseUrl}/sitemap.xml
 
-# Crawl delay for aggressive bots (optional)
-# Crawl-delay: 1
-
 # Additional resources for AI indexing
 # See https://github.com/AnswerDotAI/llms-txt for more info
 # LLM-friendly content:
 #   Manifest: ${baseUrl}/llms.txt
 #   Full content: ${baseUrl}/llms-full.txt
-#   Individual pages: ${baseUrl}/llms.mdx/[page-path]
-
-# Multi-language documentation available at:
-# ${baseUrl}/en - English
-# ${baseUrl}/es - Español
-# ${baseUrl}/fr - Français
-# ${baseUrl}/de - Deutsch
-# ${baseUrl}/ja - 日本語
-# ${baseUrl}/zh - 简体中文`
+#   Individual pages: ${baseUrl}/llms.mdx/[page-path]`
 
   return new Response(robotsTxt, {
     headers: {

diff --git a/apps/docs/app/sitemap.ts b/apps/docs/app/sitemap.ts
@@ -0,0 +1,42 @@
+import type { MetadataRoute } from 'next'
+import { i18n } from '@/lib/i18n'
+import { source } from '@/lib/source'
+import { DOCS_BASE_URL } from '@/lib/urls'
+
+export const revalidate = 3600
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const baseUrl = DOCS_BASE_URL
+  const languages = source.getLanguages()
+
+  const pagesBySlug = new Map<string, Map<string, string>>()
+  for (const { language, pages } of languages) {
+    for (const page of pages) {
+      const key = page.slugs.join('/')
+      if (!pagesBySlug.has(key)) {
+        pagesBySlug.set(key, new Map())
+      }
+      pagesBySlug.get(key)!.set(language, `${baseUrl}${page.url}`)
+    }
+  }
+
+  const entries: MetadataRoute.Sitemap = []
+  for (const [, localeMap] of pagesBySlug) {
+    const defaultUrl = localeMap.get(i18n.defaultLanguage)
+    if (!defaultUrl) continue
+
+    const langAlternates: Record<string, string> = {}
+    for (const [lang, url] of localeMap) {
+      langAlternates[lang] = url
+    }
+
+    langAlternates['x-default'] = defaultUrl
+
+    entries.push({
+      url: defaultUrl,
+      alternates: { languages: langAlternates },
+    })
+  }
+
+  return entries
+}
diff --git a/apps/docs/app/sitemap.xml/route.ts b/apps/docs/app/sitemap.xml/route.ts
diff --git a/apps/docs/components/structured-data.tsx b/apps/docs/components/structured-data.tsx
@@ -1,3 +1,5 @@
+import { DOCS_BASE_URL } from '@/lib/urls'
+
 interface StructuredDataProps {
   title: string
   description: string
@@ -15,7 +17,7 @@ export function StructuredData({
   dateModified,
   breadcrumb,
 }: StructuredDataProps) {
-  const baseUrl = 'https://docs.sim.ai'
+  const baseUrl = DOCS_BASE_URL
 
   const articleStructuredData = {
     '@context': 'https://schema.org',

diff --git a/apps/docs/lib/urls.ts b/apps/docs/lib/urls.ts
@@ -0,0 +1 @@
+export const DOCS_BASE_URL = process.env.NEXT_PUBLIC_DOCS_URL ?? 'https://docs.sim.ai'
diff --git a/apps/sim/app/layout.tsx b/apps/sim/app/layout.tsx
@@ -5,7 +5,6 @@ import { BrandedLayout } from '@/components/branded-layout'
 import { PostHogProvider } from '@/app/_shell/providers/posthog-provider'
 import { generateBrandedMetadata, generateThemeCSS } from '@/ee/whitelabeling'
 import '@/app/_styles/globals.css'
-import { OneDollarStats } from '@/components/analytics/onedollarstats'
 import { isHosted, isReactGrabEnabled, isReactScanEnabled } from '@/lib/core/config/feature-flags'
 import { HydrationErrorHandler } from '@/app/_shell/hydration-error-handler'
 import { QueryProvider } from '@/app/_shell/providers/query-provider'
@@ -207,10 +206,6 @@ export default function RootLayout({ children }: { children: React.ReactNode })
         <meta name='format-detection' content='telephone=no' />
         <meta httpEquiv='x-ua-compatible' content='ie=edge' />
 
-        {/* OneDollarStats Analytics */}
-        <link rel='dns-prefetch' href='https://assets.onedollarstats.com' />
-        <script defer src='https://assets.onedollarstats.com/stonks.js' />
-
         {/* Google Tag Manager — hosted only */}
         {isHosted && (
           <Script
@@ -260,7 +255,6 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
           </noscript>
         )}
         <HydrationErrorHandler />
-        <OneDollarStats />
         <PostHogProvider>
           <ThemeProvider>
             <QueryProvider>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		export const DOCS_BASE_URL = process.env.NEXT_PUBLIC_DOCS_URL ?? 'https://docs.sim.ai'