{
  "path": "/3mjppwxozts2f",
  "site": "at://did:plc:l37td5yhxl2irrzrgvei4qay/site.standard.publication/3m5mr6ndwbk2o",
  "tags": [],
  "$type": "site.standard.document",
  "title": "4/16/26 Service Disruption: Postmortem",
  "content": {
    "$type": "pub.leaflet.content",
    "pages": [
      {
        "id": "019d97e8-8f70-7dd4-8f08-cd17f22da1cd",
        "$type": "pub.leaflet.pages.linearDocument",
        "blocks": [
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "textSize": "small",
              "plaintext": "NOTE: This is unrelated to Bluesky’s appview outage on 4/16/26"
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "On 4/16/26, we experienced a server outage in the SE-STO-1 region, which took down our PDS and our moderation service for 15 minutes."
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "src": "https://status.blueat.net",
              "$type": "pub.leaflet.blocks.website",
              "title": "BlueAT Network status",
              "description": "Welcome to BlueAT Network status page for real-time and historical data on system performance.",
              "previewImage": {
                "$type": "blob",
                "ref": {
                  "$link": "bafkreihr4ut3rnhaiisprcwlkofeb62afmfddmwtmica63z4f2q7gtgqua"
                },
                "mimeType": "image/png",
                "size": 15997
              }
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.horizontalRule"
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "On April 16 at around 12:30 EST, we performed server maintenance to free up some disk space."
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "We deleted our /pds/pds.log to free up 3 GB of space and removed some old Docker containers, which freed about 5 GB of disk space."
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "textSize": "small",
              "plaintext": "Before we perform server maintenance we always do backups. "
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "During maintenance, we accidentally deleted our caddy configuration file while clearing Docker containers, causing our PDS and Moderation services to be inaccessible to the public."
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "Luckily, we have a public copy of our Caddy file; you can find it here. We recreated our Caddy file, pasted in the configuration, and did a quick server restart to apply the changes."
            }
          },
          {
            "$type": "pub.leaflet.pages.linearDocument#block",
            "block": {
              "$type": "pub.leaflet.blocks.text",
              "plaintext": "From now on, when we perform server maintenance, we will always double-check what commands do before running them."
            }
          }
        ]
      }
    ]
  },
  "bskyPostRef": {
    "cid": "bafyreidxzarqbcnasjga4l4u7t3gloal4xsfchmwbket7yhqflclj5zd54",
    "uri": "at://did:plc:l37td5yhxl2irrzrgvei4qay/app.bsky.feed.post/3mjppx6e56k2f",
    "commit": {
      "cid": "bafyreifmx746u2v37y5bwn6do2ok4k34r7opkstetqbogw5rx5mcdmcwse",
      "rev": "3mjppx6iwjc2k"
    },
    "validationStatus": "valid"
  },
  "description": "Today we experienced a PDS service outage that lasted 15 minutes ",
  "publishedAt": "2026-04-17T20:05:43.616Z"
}