User-agent: MJ12bot
Disallow: /

# Company-website links without "https://" turn into endless relative URLs such as
# /en/page/companies/p/4www.facebook.com...  Real listing pages are /page/companies/p/<number>.
User-agent: *
Disallow: /*/page/companies/p/*www
Disallow: /*/page/companies/p/*http
Disallow: /*/page/companies/p/*about-us
Disallow: /*/cron/
Crawl-delay: 5