Skip to content

fix(seo): use canonical SITE_URL for robots and sitemap #4598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
waleedlatif1 merged 2 commits into from
May 14, 2026
+34 −6
Merged

fix(seo): use canonical SITE_URL for robots and sitemap #4598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db4647c
fix(seo): use canonical SITE_URL for robots and sitemap
waleedlatif1 May 14, 2026
b28df2e
fix(seo): drop /templates from sitemap and guard robots/sitemap in se…
waleedlatif1 May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions apps/sim/app/(landing)/seo.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,21 @@ const SEO_SCAN_DIRS = [

const SEO_SCAN_INDIVIDUAL_FILES = [
path.resolve(APP_DIR, 'page.tsx'),
path.resolve(APP_DIR, 'robots.ts'),
path.resolve(APP_DIR, 'sitemap.ts'),
path.resolve(SIM_ROOT, 'ee', 'whitelabeling', 'metadata.ts'),
]

/**
* Files whose entire URL output is SEO-facing (robots.txt, sitemap.xml).
* Unlike metadata exports, these don't use `metadataBase`, so the existing
* `getBaseUrl()`-in-metadata check would miss a regression here.
*/
const SEO_DEFAULT_EXPORT_FILES = [
path.resolve(APP_DIR, 'robots.ts'),
path.resolve(APP_DIR, 'sitemap.ts'),
]

function collectFiles(dir: string, exts: string[]): string[] {
const results: string[] = []
if (!fs.existsSync(dir)) return results
Expand Down Expand Up @@ -97,6 +109,21 @@ describe('SEO canonical URLs', () => {
).toHaveLength(0)
})

it('robots.ts and sitemap.ts do not import getBaseUrl', () => {
const violations: string[] = []
for (const file of SEO_DEFAULT_EXPORT_FILES) {
if (!fs.existsSync(file)) continue
const content = fs.readFileSync(file, 'utf-8')
if (content.includes('getBaseUrl')) {
violations.push(path.relative(SIM_ROOT, file))
}
}
expect(
violations,
`robots.ts/sitemap.ts must use SITE_URL, not getBaseUrl():\n${violations.join('\n')}`
).toHaveLength(0)
})

it('public pages do not use getBaseUrl() for SEO metadata', () => {
const files = getAllSeoFiles(['.ts', '.tsx'])
const violations: string[] = []
Expand Down
6 changes: 2 additions & 4 deletions apps/sim/app/robots.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { MetadataRoute } from 'next'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { SITE_URL } from '@/lib/core/utils/urls'

const DISALLOWED_PATHS = [
'/api/',
Expand Down Expand Up @@ -45,8 +45,6 @@ const LINK_PREVIEW_BOTS = [
]

export default function robots(): MetadataRoute.Robots {
const baseUrl = getBaseUrl()

return {
rules: [
{ userAgent: '*', allow: '/', disallow: DISALLOWED_PATHS },
Expand All @@ -56,6 +54,6 @@ export default function robots(): MetadataRoute.Robots {
disallow: LINK_PREVIEW_DISALLOWED_PATHS,
},
],
sitemap: [`${baseUrl}/sitemap.xml`, `${baseUrl}/blog/sitemap-images.xml`],
sitemap: [`${SITE_URL}/sitemap.xml`, `${SITE_URL}/blog/sitemap-images.xml`],
}
}
7 changes: 5 additions & 2 deletions apps/sim/app/sitemap.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import type { MetadataRoute } from 'next'
import { COURSES } from '@/lib/academy/content'
import { getAllPostMeta } from '@/lib/blog/registry'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { SITE_URL } from '@/lib/core/utils/urls'
Comment thread
waleedlatif1 marked this conversation as resolved.
import integrations from '@/app/(landing)/integrations/data/integrations.json'
import { ALL_CATALOG_MODELS, MODEL_PROVIDERS_WITH_CATALOGS } from '@/app/(landing)/models/utils'

export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
const baseUrl = getBaseUrl()
const baseUrl = SITE_URL
const posts = await getAllPostMeta()

const latestPostDate =
Expand Down Expand Up @@ -46,6 +46,9 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
{
url: `${baseUrl}/partners`,
},
{
url: `${baseUrl}/contact`,
},
{
url: `${baseUrl}/terms`,
lastModified: new Date('2024-10-14'),
Expand Down
Loading