# copy from https://www.netflix.com/robots.txt
User-agent: googlebot
User-agent: Googlebot-Video
User-agent: Applebot
User-agent: bingbot
User-agent: NaverBot
User-agent: Yeti
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexMobileBot
User-agent: YandexVideo
User-agent: YandexWebmaster
User-agent: YandexSitelinks
User-agent: SeznamBot
Allow: /
Allow: /docs
Allow: /posts

User-agent: AdsBot-Google
User-agent: Twitterbot
User-agent: Adidxbot
Allow: /

User-agent: Yisouspider
User-agent: Sogou inst spider
User-agent: Sogou web spider
User-agent: 360Spider
User-agent: Bytespider
User-agent: PetalBot
User-agent: Baiduspider
User-agent: Baiduspider-mobile
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /