# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /

# Excluding assets folder for all bots
User-agent: *
Disallow: /assets/

# Allowing twitter bot for play urls
User-agent: Twitterbot
Allow: /video/play/

# Blocking facebook bots
User-Agent: FacebookBot
Disallow: /*

User-Agent: meta-externalagent
Disallow: /*

# Allowing twitter bot for play urls
User-agent: Twitterbot
Allow: /video/embed/

# Excluding video delivery page for all bots
User-agent: *
Disallow: /video/play/

# Excluding sales page indexing for all bots
User-agent: *
Disallow: /page/


User-agent: Googlebot
Allow: /video/embed/

# Excluding video embed page for all bots
User-agent: *
Disallow: /video/embed/

#Ahref
User-agent: AhrefsBot
Crawl-Delay: 120


Sitemap: https://www.hippovideo.io/sitemap.xml