# www.my-ads.co.uk User-agent: * Disallow: /cgi-bin/ Disallow: /images/ Disallow: /group/ Disallow: /phpSitemapNG/ Disallow: /unused/ Disallow: /style/ Disallow: /bbclone/ Disallow: /image/ #User-agent: msnbot #Crawl-delay: 10 #User-agent: Teoma #Crawl-delay: 10 #User-agent: Slurp #Crawl-delay: 10 User-agent: aipbot Disallow: / User-agent: BecomeBot Disallow: / User-agent: psbot Disallow: / #This is an example robots.txt file User-agent: TurnitinBot Disallow: / #Will disallow all urls on your site