User-Agent: *
Disallow: /cron/
Disallow: /server/
Disallow: /insert
Disallow: /worker/
Allow: /