Bootstrap/site/robots.txt

---
---

# www.robotstxt.org/

{% if jekyll.environment != "netlify" -%}
# Allow crawling of all content
{%- endif %}
User-agent: *
Disallow:{% if jekyll.environment == "netlify" %} /{% endif %}
Sitemap: {{ site.url }}/sitemap.xml
Add sitemap.xml and robots.txt files. 2014-02-23 12:33:33 +01:00			`---`
			`---`

			`# www.robotstxt.org/`

Disallow crawling on Netlify. (#29194) 2019-08-03 16:09:25 +02:00			`{% if jekyll.environment != "netlify" -%}`
Add sitemap.xml and robots.txt files. 2014-02-23 12:33:33 +01:00			`# Allow crawling of all content`
Disallow crawling on Netlify. (#29194) 2019-08-03 16:09:25 +02:00			`{%- endif %}`
Add sitemap.xml and robots.txt files. 2014-02-23 12:33:33 +01:00			`User-agent: *`
Disallow crawling on Netlify. (#29194) 2019-08-03 16:09:25 +02:00			`Disallow:{% if jekyll.environment == "netlify" %} /{% endif %}`
Add sitemap.xml and robots.txt files. 2014-02-23 12:33:33 +01:00			`Sitemap: {{ site.url }}/sitemap.xml`