fix(deploy): atomic deploys — build in builds/<sha-stamp>, verify fresh dist, swap, health-check + auto-rollback

- Stage to timestamped builds/ dir; live custom-build untouched during install+build
- Verify dist exists, non-empty, newer than stage start (stale/partial fails)
- Symlink swap stays single atomic op; previous target saved for rollback
- Post-restart health-check on / (24x5s); on failure flip back, restart, notify, exit 1
- Prune to 2 newest builds on success; notify on failure (if: failure())
- Root cause: rm -rf custom-build destroyed the live dir mid-deploy; UI 404'd until build finished

.gitea/workflows/deploy-custom.yml

@@ -8,26 +8,78 @@ jobs:
     steps:
       - name: Checkout
         run: |
+          set -euo pipefail
           rm -rf $GITHUB_WORKSPACE/repo
           mkdir -p $GITHUB_WORKSPACE/repo
           git clone --depth 1 --branch custom gitea@giteassh.buzzbee.dev:BuzzbeeSCD/openchamber.git $GITHUB_WORKSPACE/repo
       - name: Show revision
         run: git -C $GITHUB_WORKSPACE/repo rev-parse --short HEAD
-      - name: Copy workspace to build dir
+      - name: Stage new build (atomic — live dir untouched)
         run: |
-          rm -rf /opt/app/deploy/custom-build
-          mkdir -p /opt/app/deploy/custom-build
-          tar -C $GITHUB_WORKSPACE/repo --exclude=.git -cf - . | tar -C /opt/app/deploy/custom-build -xf -
+          set -euo pipefail
+          SHA=$(git -C $GITHUB_WORKSPACE/repo rev-parse --short HEAD)
+          STAMP=$(date +%Y%m%d-%H%M%S)
+          NEWDIR=/opt/app/deploy/builds/${SHA}-${STAMP}
+          mkdir -p "$NEWDIR"
+          touch "$NEWDIR/.stage-start"
+          tar -C $GITHUB_WORKSPACE/repo --exclude=.git -cf - . | tar -C "$NEWDIR" -xf -
+          echo "$NEWDIR" > $GITHUB_WORKSPACE/newdir
+          echo "staged $NEWDIR"
       - name: Install dependencies
-        run: cd /opt/app/deploy/custom-build && bun install --frozen-lockfile
+        run: |
+          set -euo pipefail
+          NEWDIR=$(cat $GITHUB_WORKSPACE/newdir)
+          cd "$NEWDIR" && bun install --frozen-lockfile
       - name: Build web
-        run: cd /opt/app/deploy/custom-build && bun run build:web
-      - name: Verify dist
-        run: test -f /opt/app/deploy/custom-build/packages/web/dist/index.html
-      - name: Swap current symlink
-        run: ln -sfn /opt/app/deploy/custom-build /opt/app/deploy/current
+        run: |
+          set -euo pipefail
+          NEWDIR=$(cat $GITHUB_WORKSPACE/newdir)
+          cd "$NEWDIR" && bun run build:web
+      - name: Verify dist (fresh, non-empty — stale/partial fails here)
+        run: |
+          set -euo pipefail
+          NEWDIR=$(cat $GITHUB_WORKSPACE/newdir)
+          DIST="$NEWDIR/packages/web/dist/index.html"
+          test -f "$DIST"
+          test -s "$DIST"
+          test "$DIST" -nt "$NEWDIR/.stage-start"
+          rm -f "$NEWDIR/.stage-start"
+          echo "dist verified: $(stat -c '%s bytes, %y' "$DIST")"
+      - name: Swap current symlink (single atomic op)
+        run: |
+          set -euo pipefail
+          NEWDIR=$(cat $GITHUB_WORKSPACE/newdir)
+          readlink /opt/app/deploy/current > $GITHUB_WORKSPACE/prevdir
+          ln -sfn "$NEWDIR" /opt/app/deploy/current
+          echo "swapped: $(cat $GITHUB_WORKSPACE/prevdir) -> $NEWDIR"
       - name: Restart service
         run: sudo systemctl restart openchamber-custom
+      - name: Health-check (auto-rollback on failure)
+        run: |
+          set -euo pipefail
+          NEWDIR=$(cat $GITHUB_WORKSPACE/newdir)
+          PREVDIR=$(cat $GITHUB_WORKSPACE/prevdir)
+          SHA=$(git -C $GITHUB_WORKSPACE/repo rev-parse HEAD)
+          ok=0
+          for i in $(seq 1 24); do
+            CODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3001/ || true)
+            if [ "$CODE" = "200" ]; then ok=1; break; fi
+            sleep 5
+          done
+          if [ "$ok" = "1" ]; then
+            echo "healthy on $NEWDIR — pruning old builds"
+            ls -dt /opt/app/deploy/builds/*/ 2>/dev/null | tail -n +3 | xargs -r rm -rf
+            exit 0
+          fi
+          echo "HEALTH CHECK FAILED — rolling back to $PREVDIR"
+          ln -sfn "$PREVDIR" /opt/app/deploy/current
+          sudo systemctl restart openchamber-custom
+          sleep 15
+          RCODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3001/ || true)
+          MSG="OpenChamber deploy FAILED (health $RCODE after rollback to $PREVDIR)"
+          if [ "$RCODE" = "200" ]; then MSG="$MSG — rollback healthy"; else MSG="$MSG — ROLLBACK ALSO UNHEALTHY, manual intervention needed"; fi
+          /opt/app/.local/bin/openchamber-notify-deploy.sh "$SHA" "$MSG" "deploy-bot"
+          exit 1
 
       - name: Notify on successful deploy
         run: |
@@ -35,3 +87,9 @@ jobs:
           MSG=$(git -C $GITHUB_WORKSPACE/repo log -1 --format=%s)
           AUTHOR=$(git -C $GITHUB_WORKSPACE/repo log -1 --format=%an)
           /opt/app/.local/bin/openchamber-notify-deploy.sh "$SHA" "$MSG" "$AUTHOR"
+
+      - name: Notify on failed deploy
+        if: failure()
+        run: |
+          SHA=$(git -C $GITHUB_WORKSPACE/repo rev-parse HEAD || echo unknown)
+          /opt/app/.local/bin/openchamber-notify-deploy.sh "$SHA" "OpenChamber deploy FAILED before health-check (see Actions log)" "deploy-bot"