User-agent: *
Allow: /

# Sitemaps
Sitemap: https://fisip-umw.ac.id/sitemap.xml

# No crawl delay - allow fast crawling
# Crawl-delay: 0

# Disallow admin areas (if any)
Disallow: /admin/
Disallow: /private/
Disallow: /*.json$
Disallow: /api/

# Allow important pages
Allow: /about
Allow: /programs  
Allow: /contact
Allow: /teachers

# Encourage frequent crawling
User-agent: AhrefsBot
Allow: /
Crawl-delay: 0

User-agent: SemrushBot  
Allow: /
Crawl-delay: 0