User-agent: *
Crawl-delay: 30
Disallow: /announcements/checkShared/*
Disallow: /announcements/fetchFacebookComments/*
Disallow: /admin/*           # Disallow sensitive sections
Disallow: /login/           # Prevent crawling login pages
Allow: /

User-Agent: MJ12bot
Crawl-Delay: 15