Sitemap: http://www.open24.pl/sitemap/6/sitemap.xml

User-agent: *
Disallow: /admin/


User-agent: Bingbot #Bing crawler
crawl-delay: 30

User-agent: Slurp #yahoo! crawler
crawl-delay: 30

User-agent: DuckDuckBot #DuckDuckGo crawler
crawl-delay: 30

User-agent: Baiduspider #Baiduspider crawler
crawl-delay: 30

User-agent: YandexBot #Yandex crawler
crawl-delay: 30

User-agent: ia_archiver #Alexa crawler
crawl-delay: 30