# www.newbuildhomes.org robots.txt

User-agent: *

# Allow all bots to crawl most paths
Allow: /
Allow: /build-to-rent
Allow: /for-sale
Allow: /shared-ownership
Allow: /developers
Allow: /london
Allow: /*/properties

# Prevent crawling of search results and filters
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*search=
Disallow: /*?*price=
Disallow: /*?*beds=
Disallow: /search
Disallow: /api/
Disallow: /admin/

# Prevent crawling of authentication paths
Disallow: /login
Disallow: /signup
Disallow: /account

# Allow crawling of sitemap
Sitemap: https://www.newbuildhomes.org/sitemap.xml

# Crawl delay to prevent server overload
Crawl-delay: 1

# Specific rules for major bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1