{
  "$type": "site.standard.document",
  "bskyPostRef": {
    "cid": "bafyreigzhxvegli6svs2udfzp7hezuttvgje25663lzn46xq57rctmbnam",
    "uri": "at://did:plc:ixd3llanirxy6by2epvxdu7k/app.bsky.feed.post/3mlu7citiwcb2"
  },
  "description": "About time! One down, twenty more to go (give or take 1000)\n\nhttps://xeiaso.net/notes/2026/amazonbot-respecting-robots-txt/\n\n@cadey@pony.social does this mean default bot policy will now let them through? Neat.",
  "path": "/post/1879",
  "publishedAt": "2026-05-15T01:41:35.822Z",
  "site": "https://activitypub.space",
  "tags": [
    "llm",
    "anubis",
    "@cadey@pony.social",
    "https://xeiaso.net/notes/2026/amazonbot-respecting-robots-txt/"
  ],
  "textContent": "About time! One down, twenty more to go (give or take 1000)\n\nhttps://xeiaso.net/notes/2026/amazonbot-respecting-robots-txt/\n\n@cadey@pony.social does this mean default bot policy will now let them through? Neat.",
  "title": "Xeiaso blog: Amazonbot is finally respecting robots.txt"
}