User-agent: * Disallow: /cgi-bin Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /wp-cron.php Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins Disallow: /wp-content/cache Disallow: /wp-content/themes Disallow: /webstat/ Disallow: /feed/ Disallow: /tag/ Disallow: /wp-json/ Disallow: /search/ Disallow: /goto/ Disallow: *.html/* Disallow: */sphinx/* Disallow: */*.js* Disallow: /daos/ Disallow: */?p* Disallow: */?s* Crawl-delay: 60 Host: www.ws-craft.ru Sitemap: https://www.ws-craft.ru/sitemap.xml.gz Sitemap: https://www.ws-craft.ru/sitemap.xml User-agent: Yandex Disallow: /downloads/ Request-rate: 1/60 Crawl-delay: 60 User-agent: SetLinks bot 2.0 Crawl-delay: 99 Disallow: /downloads/ Request-rate: 1/1515 User-agent: SetLinks Crawl-delay: 99 Disallow: /downloads/ Request-rate: 1/1515 User-agent: SetLinks bot Crawl-delay: 99 Disallow: /downloads/ Request-rate: 1/1515 User-agent: vkShare Disallow: /downloads/ Request-rate: 1/15800 Crawl-delay: 998111 User-agent: mlbot Disallow: /downloads/ User-agent: MSIE Disallow: /downloads/ Request-rate: 1/505 Crawl-delay: 600 User-agent: YandexBot3.0 Disallow: /downloads/ Request-rate: 1/60 Crawl-delay: 60 User-agent: YandexBot/3.0 Disallow: /downloads/ Request-rate: 1/60 Crawl-delay: 60 User-agent: Twitterbot/1.0 Disallow: /downloads/ Request-rate: 1/60000 Crawl-delay: 60000 User-agent: Twitterbot Disallow: /downloads/ Request-rate: 1/60000 Crawl-delay: 60000 User-agent: Twitter Disallow: /downloads/ Request-rate: 1/60000 Crawl-delay: 60000 User-agent: Riddler Disallow: / User-agent: Slurp Disallow: / User-agent: MSNBot/Bingbot Crawl-delay: 10000