Bootstrap/docs/robots.txt

---
---

# www.robotstxt.org/

{% if site.netlify != true and site.github == true -%}
# Allow crawling of all content
{%- endif %}
User-agent: *
Disallow:{% if site.netlify == true or site.github != true %} /{% endif %}
Sitemap: {{ site.url }}/sitemap.xml
Add sitemap.xml and robots.txt files. 2014-02-23 13:33:33 +02:00			`---`
			`---`

			`# www.robotstxt.org/`

Disable crawling on Netlify. (#29193) 2019-08-03 17:09:03 +03:00			`{% if site.netlify != true and site.github == true -%}`
Add sitemap.xml and robots.txt files. 2014-02-23 13:33:33 +02:00			`# Allow crawling of all content`
Disable crawling on Netlify. (#29193) 2019-08-03 17:09:03 +03:00			`{%- endif %}`
Add sitemap.xml and robots.txt files. 2014-02-23 13:33:33 +02:00			`User-agent: *`
Disable crawling on Netlify. (#29193) 2019-08-03 17:09:03 +03:00			`Disallow:{% if site.netlify == true or site.github != true %} /{% endif %}`
Add sitemap.xml and robots.txt files. 2014-02-23 13:33:33 +02:00			`Sitemap: {{ site.url }}/sitemap.xml`