User-agent: * Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ # Explicitly allow major AI/search crawlers to fetch public, prerendered HTML. # Markdown mirrors and llms.txt are public so agents can read concise page content. # Private app areas remain disallowed for every crawler group below. User-agent: GPTBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: OAI-SearchBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: ChatGPT-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: ClaudeBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Claude-SearchBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Claude-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: PerplexityBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Perplexity-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Googlebot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Google-Extended Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Bingbot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: DuckDuckBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: Applebot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ User-agent: CCBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /auth Disallow: /auth/ Disallow: /candidate Disallow: /candidate/ Disallow: /employer Disallow: /employer/ Disallow: /admin Disallow: /admin/ Disallow: /role-select Disallow: /role-select/ Disallow: /unsubscribe Disallow: /unsubscribe/ Sitemap: https://talent-poacher.com/sitemap.xml