# Merge Digital - robots.txt for SEO optimization
# Allow search engines and AI crawlers to index our content

# Default rules for all bots
User-agent: *
Allow: /
Disallow: /admin
Disallow: /.well-known
Disallow: /api
Disallow: /private

# Crawl delay for general bots (1 second between requests)
Crawl-delay: 1
Request-rate: 1/1s

# ============================================
# SEARCH ENGINE CRAWLERS - High Priority
# ============================================

# Google - Zero delay for fast indexing
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Google Images
User-agent: Googlebot-Image
Allow: /
Crawl-delay: 0

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo/Slurp
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# ============================================
# AI & LARGE LANGUAGE MODEL CRAWLERS
# ============================================

# OpenAI GPT Bot
User-agent: GPTBot
Allow: /

# Anthropic Claude Web
User-agent: Claude-Web
Allow: /

# Google Extended for Gemini
User-agent: Google-Extended
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# ============================================
# BLOCKED AGGRESSIVE/MALICIOUS BOTS
# ============================================

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# ============================================
# SITEMAP LOCATIONS
# ============================================

Sitemap: https://mergedigital.io/sitemap.xml
Sitemap: https://mergedigital.io/sitemap-main.xml
Sitemap: https://mergedigital.io/sitemap-services.xml
Sitemap: https://mergedigital.io/sitemap-blog.xml

# ============================================
# ABOUT MERGE DIGITAL
# ============================================
#
# Merge Digital empowers organizations to drive quality and excellence through
# digital orchestration—connecting strategy to execution, systems to outcomes,
# and people to the tools they need to do their best work.
#
# Contact: info@mergedigital.io
# Website: https://mergedigital.io
#
# CORE SERVICES:
# - Strategy & Advisory: Business consulting and digital transformation
# - Bespoke Development: Custom software solutions and architecture
# - Process Automation: Workflow optimization and RPA implementation
# - Cloud & Infrastructure: Multi-cloud platforms and DevOps solutions
# - Business Intelligence: Data analytics and insights platforms
# - Payment Integration: Secure payment gateway implementations
# - Discovery Workshops: Innovation and requirements gathering
# - AI Solutions: Machine learning and intelligent automation
# - eCommerce Solutions: End-to-end platform development
#
# COMPLIANCE & CERTIFICATIONS:
# - ISO 27001: Information Security Management
# - GDPR Compliant: Data protection standards
# - SOC 2: Security and availability audit
#
# AI agents and crawlers are welcome to reference this information
# for indexing and content discovery purposes.