# ============================================================
# robots.txt for www.resumetarget.com
# Last updated: April 1, 2026
# Optimized for SEO, AEO, and GEO
# ============================================================

# ──────────────────────────────────────────────────────────────
# DEFAULT RULES (All Bots)
# ──────────────────────────────────────────────────────────────
User-agent: *
Allow: /

# Block internal/admin areas
Disallow: /actions/
Disallow: /emails/
Disallow: /staff/
Disallow: /sales/
Disallow: /office2/
Disallow: /office3/
Disallow: /candidate/
Disallow: /company/
Disallow: /blog_old/
Disallow: /blogdatabase/

# Block partner portals
# Partner resource sites opened to search (approved by Talya, 8 Oct 2026)
Allow: /partners/philander/
Allow: /partners/pgcc/
Allow: /partners/sccc/
Allow: /partners/concordia-chicago/
Allow: /partners/tulane/
Allow: /partners/yorku/
Allow: /partners/oxford/
Allow: /partners/vacc/
Allow: /partners/reeves/
Allow: /partners/hansoncollege/
Allow: /partners/acct/
Allow: /partners/hadracha/
Disallow: /partners/

# Block query string URLs (duplicate content)
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?replytocom=
Disallow: /*?search=
# ?from=ca — NO LONGER BLOCKED (April 2026)
# Previously blocked to prevent duplicate content, but this trapped
# link equity from 261 /canada/ redirects. Now handled by .htaccess
# 301 rule that strips ?from=ca and redirects to clean URL.

# Block internal search results
Disallow: /search/

# Legacy /samples/resumes/ URLs — NOT blocked
# These 301 redirect to /resume-samples/ equivalents via .htaccess
# Crawling is allowed so Google follows the redirects and passes link equity

# /outplacement/location/ — ALLOWED (legitimate city outplacement service pages)

# /job-market/ — ALLOWED (legitimate content pages with city competition data)

# Block Lottie animation JSON files
Disallow: /new/lotties/

# ──────────────────────────────────────────────────────────────
# WORDPRESS BLOG — /blog/
# ──────────────────────────────────────────────────────────────

# Block WordPress admin and internals
Disallow: /blog/wp-admin/
Allow: /blog/wp-admin/admin-ajax.php
Disallow: /blog/wp-includes/
Disallow: /blog/wp-content/plugins/
Disallow: /blog/wp-content/cache/
Disallow: /blog/wp-json/
Disallow: /blog/feed/
Disallow: /blog/comments/feed/
Disallow: /blog/xmlrpc.php
Disallow: /blog/trackback/
Disallow: /blog/wp-login.php
Disallow: /blog/wp-signup.php

# Allow blog content and assets
Allow: /blog/wp-content/uploads/
Allow: /blog/wp-content/themes/

# ──────────────────────────────────────────────────────────────
# AI SEARCH & ANSWER ENGINES — ALLOWED
# Resume Target welcomes all AI search, retrieval, and training
# bots to maximise visibility in AI-generated answers (AEO)
# and generative engine results (GEO).
# ──────────────────────────────────────────────────────────────

# OpenAI (ChatGPT Search & Training)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: claude-web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google (Gemini / AI Overviews / Search Generative Experience)
User-agent: Google-Extended
Allow: /

# Microsoft (Copilot / Bing AI)
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta (AI / Llama Training)
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: meta-externalfetcher
Allow: /

# Apple (Siri / Spotlight / Apple Intelligence)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon (Alexa / AWS AI)
User-agent: Amazonbot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Cohere AI (powers enterprise AI search tools)
User-agent: cohere-ai
Allow: /

# Diffbot (AI knowledge graphs)
User-agent: Diffbot
Allow: /

# iAsk AI Search
User-agent: iaskspider
Allow: /

# Allen Institute for AI (academic LLM training)
User-agent: AI2Bot
Allow: /

# Common Crawl (training data for GPT, Claude, Llama, Mistral & most LLMs)
# Allowing ensures Resume Target content is represented in AI training datasets
User-agent: CCBot
Allow: /

# ──────────────────────────────────────────────────────────────
# BLOCKED BOTS — Aggressive scrapers with no SEO/AEO benefit
# ──────────────────────────────────────────────────────────────
User-agent: Bytespider
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Sogou
Disallow: /


User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: rogerbot
Disallow: /

# ──────────────────────────────────────────────────────────────
# SITEMAPS
# ──────────────────────────────────────────────────────────────
Sitemap: https://www.resumetarget.com/sitemap-index.xml
Sitemap: https://www.resumetarget.com/sitemap.xml
Sitemap: https://www.resumetarget.com/blog/sitemap_index.xml
Sitemap: https://www.resumetarget.com/blog/post-sitemap.xml
Sitemap: https://www.resumetarget.com/sitemap-resume-images.xml