User-agent: AhrefsBot Crawl-Delay: 5 # don't allow search bots to index css, javascript and images User-agent: * Disallow: /css/ Disallow: /js/ Disallow: /img/ Disallow: /ws/download/