diff --git a/robots.txt b/robots.txt index 690230f4f6aa0508aeff8ce29faeef35b866d347..a40c2e00a02676028a3fd2c35781d2df30eca96d 100644 --- a/robots.txt +++ b/robots.txt @@ -4,6 +4,9 @@ sitemap: exclude: 'yes' --- User-agent: * -Disallow: +Disallow: /search + +User-agent: ia_archiver +Disallow: / Sitemap: {{ site.url }}/sitemap.xml