# SEOWebster Multi-Country Robots.txt
# Main site and country-specific pages

User-agent: *
Allow: /

# Main site pages
Allow: /services/
Allow: /cms-seo/
Allow: /industry-seo/
Allow: /blog/
Allow: /contact
Allow: /shop
Allow: /partners

# Country-specific pages
Allow: /lk/
Allow: /lk/services/
Allow: /lk/cms-seo/
Allow: /lk/seo-services/
Allow: /lk/contact

Allow: /au/
Allow: /au/services/
Allow: /au/cms-seo/
Allow: /au/seo-services/
Allow: /au/contact

# Block admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /_astro/
Disallow: /node_modules/
Disallow: /.git/

# Allow all sitemaps
Sitemap: https://seowebster.com/sitemap-index.xml
Sitemap: https://seowebster.com/sitemap.xml
Sitemap: https://seowebster.com/sitemap-countries.xml
Sitemap: https://seowebster.com/sitemap-pages.xml
Sitemap: https://seowebster.com/sitemap-services.xml
Sitemap: https://seowebster.com/sitemap-content.xml

# Crawl delay for respectful crawling
Crawl-delay: 1

# Country-specific comments for search engines
# Sri Lanka (LK) specific content optimized for local search
# Australia (AU) specific content optimized for Australian market
# United States (US) default content for international audience