# =============================================================================
# robots.txt for agardco.com
# Ågård & Co AB — Privatägt investmentbolag inom bygg och fastighet
# Updated: 2026-08-24
#
# AI/LLM resources:
#   https://agardco.com/llms.txt       (concise index, llmstxt.org format)
#   https://agardco.com/llms-full.txt  (full structured company information)
#   https://agardco.com/ai-policy      (AI usage policy)
# =============================================================================

# -----------------------------------------------------------------------------
# Default rules for all crawlers
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /.well-known/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /img/
Allow: /css/
Allow: /js/
Disallow: /includes/
Disallow: /admin/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /*.log
Disallow: /*.sql
Disallow: /*.bak
Disallow: /*.env

# -----------------------------------------------------------------------------
# Search engines
# -----------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /img/

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# -----------------------------------------------------------------------------
# AI crawlers — training (allowed, see /ai-policy and /llms-full.txt)
# -----------------------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /includes/

User-agent: ClaudeBot
Allow: /
Disallow: /includes/

User-agent: anthropic-ai
Allow: /
Disallow: /includes/

User-agent: Google-Extended
Allow: /
Disallow: /includes/

User-agent: Applebot-Extended
Allow: /
Disallow: /includes/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /includes/

User-agent: Amazonbot
Allow: /
Disallow: /includes/

User-agent: CCBot
Allow: /
Disallow: /includes/

User-agent: cohere-ai
Allow: /
Disallow: /includes/

User-agent: MistralAI-User
Allow: /
Disallow: /includes/

# -----------------------------------------------------------------------------
# AI crawlers — search & live retrieval (allowed)
# -----------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# -----------------------------------------------------------------------------
# SEO tool crawlers — throttled
# -----------------------------------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: Bytespider
Crawl-delay: 10

# -----------------------------------------------------------------------------
# Blocked crawlers — no value, high load or scraping tools
# -----------------------------------------------------------------------------
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Wget
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

# -----------------------------------------------------------------------------
# Sitemaps
# -----------------------------------------------------------------------------
Sitemap: https://agardco.com/sitemap.xml

# =============================================================================
# Analytics: Besökskollen (https://besokskollen.se) by Savri (https://savri.io)
# — privacy-friendly, cookie-free web analytics
# End of robots.txt
# =============================================================================