{
  "$type": "site.standard.document",
  "bskyPostRef": {
    "cid": "bafyreig2d276rl54bjcqv6mqf2eb3swqbwfypudnfwcnuuensx7q27eniq",
    "uri": "at://did:plc:hqad6xwuzg7oqfmwylfkvqfm/app.bsky.feed.post/3mlzpimykfib2"
  },
  "path": "/viewtopic.php?t=32045&p=274189#p274189",
  "publishedAt": "2026-05-17T05:37:57.000Z",
  "site": "http://forum.palemoon.org",
  "textContent": "> robots.txt\n\nAI scrapers don't honor robots.txt at all.\n\nUA filtering is pointless as they go out of their way to use a smattering of Chrome UAs when scraping, and do not identify as bots in the UA at all.\n\nExample of a scraper UA:\n\nCODE:\n\n\n    Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36\n\n* * *",
  "title": "Browser Support • Re: Cloudflare Verification Loop issues",
  "updatedAt": "2026-05-17T05:37:57.000Z"
}