User-agent: *
Crawl-Delay: 10
Disallow: /superadmin/
Disallow: /pub/
Disallow: /admin/
Disallow: /images/
Disallow: /titles/
Disallow: /content/
Disallow: /users/
Disallow: /cgi-bin/
Disallow: /buydomain.cgi
Disallow: /buybanner.cgi
Disallow: /longterm.cgi
Disallow: /mpr.cgi
Disallow: /paypal.cgi
Disallow: /user.cgi
Disallow: /marketplace.cgi
Disallow: /demo/
Allow: /

User-agent: Website eXtractor 
Disallow:/

User-agent: Wget 
Disallow:/