gitoriaLog in with ident

tracker

All repositories: gitoria

ReadmeCodePull requestsReleasesTicketsSettingsCommit5cb85d755cb85d75deploy.sh: a backup taken while a background job writes (tar exit 1) is a warning; archive checked with gzip -tmre5cb85d75/deploy.sh

9.5 KB

  1. #!/usr/bin/env bash
  2. # deploy.sh — tracker.worldapi.org: Loreana (this folder) → Byrodin. Run ON LOREANA.
  3. #
  4. # ./deploy.sh gates → backup → rsync → restart the container → public URL 200
  5. # ./deploy.sh --dry-run gates → rsync -n (shows what WOULD be sent), no backup, no restart, no URL check
  6. # ./deploy.sh --skip-tests skips the gates (LOUD warning) — only when you know why
  7. # ./deploy.sh --target DIR|HOST:DIR another destination (default below); a local DIR is
  8. # how the script is tested without touching Byrodin
  9. # ./deploy.sh --url URL the URL that must answer 200 after the restart (asked for up to 90 s)
  10. # (env: DEPLOY_TARGET, DEPLOY_URL, DEPLOY_SSH override the same defaults; DEPLOY_URL_WAIT the seconds the URL may take)
  11. # (backup: tars storage/.sessions/.env that exist on the target to Loreana's
  12. # /media/SLOW1TB2/deploy-backups/<app>/, keeps newest 5; --target DIR backs up DIR instead)
  13. #
  14. # THE FIRST DEPLOY is done by the architect on Byrodin (folder, .env with this app's ident
  15. # API key + secret — registered once in ident's /apps for origin https://tracker.worldapi.org
  16. # — nginx vhost, cert, DNS). This script only updates the CODE: storage/, .sessions/, .env,
  17. # .scratch/, server.*, tests/ fixtures and logs are never sent, so live data on Byrodin is
  18. # never touched. No --delete: a file removed here stays on Byrodin (harmless).
  19. set -euo pipefail
  21. # ---- the app ------------------------------------------------------------------------------
  22. APP=tracker.worldapi.org
  23. CONTAINER=tracker.worldapi.org
  24. [email protected]:/CONTAINERS/projects/tracker.worldapi.org
  25. DEFAULT_URL=https://tracker.worldapi.org/
  26. GATES=(
  27. "node tests/browser.mjs"
  28. "node tests/kinds.mjs"
  29. "node tests/franchises.mjs"
  30. "node tests/pager.mjs"
  31. "node tests/franchiseseed.mjs"
  32. "node theme/check-theme.mjs ."
  33. )
  34. # NEVER SENT (rsync patterns; a leading / anchors at the app folder)
  35. EXCLUDES=(
  36. /.git/ /.gitignore
  37. /storage/ /.sessions/ /.env /.env.* /.scratch/ /server.* '*.log' '*.pid' node_modules/
  38. )
  39. # ---------------------------------------------------------------------------------------------
  41. DRY=0
  42. SKIP=0
  43. TARGET=${DEPLOY_TARGET:-$DEFAULT_TARGET}
  44. URL=${DEPLOY_URL:-$DEFAULT_URL}
  45. # Loreana's ssh config may not apply to Byrodin: -F /dev/null (ident STATUS)
  46. SSH=${DEPLOY_SSH:-ssh -F /dev/null -o BatchMode=yes -o ConnectTimeout=15}
  48. usage() { sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; }
  49. while [ $# -gt 0 ]; do
  50. case "$1" in
  51. --dry-run) DRY=1 ;;
  52. --skip-tests) SKIP=1 ;;
  53. --target) TARGET=${2:?--target needs a value}; shift ;;
  54. --target=*) TARGET=${1#--target=} ;;
  55. --url) URL=${2:?--url needs a value}; shift ;;
  56. --url=*) URL=${1#--url=} ;;
  57. -h|--help) usage; exit 0 ;;
  58. *) echo "deploy: unknown argument: $1" >&2; usage >&2; exit 2 ;;
  59. esac
  60. shift
  61. done
  63. HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
  64. cd "$HERE"
  66. step() { printf '\n==> %s\n' "$*"; }
  67. run() { printf ' $ %s\n' "$*" >&2; "$@"; }
  68. die() { printf '\ndeploy: REFUSED — %s\n' "$*" >&2; exit 1; }
  70. # a target `host:dir` is remote, a plain path is local
  71. if [[ "$TARGET" == *:* ]]; then
  72. REMOTE_HOST=${TARGET%%:*}
  73. REMOTE_DIR=${TARGET#*:}
  74. else
  75. REMOTE_HOST=
  76. REMOTE_DIR=$TARGET
  77. fi
  79. step "[0/5] $APP → $TARGET (dry run: $DRY, skip tests: $SKIP)"
  80. echo " from $HERE"
  81. echo " url $URL"
  82. [ -x bin/hybriel ] || die "bin/hybriel is missing here"
  83. [ -f docker-compose.yml ] || die "docker-compose.yml is missing here"
  84. [ -f project.hl ] || die "project.hl is missing here"
  85. command -v rsync >/dev/null || die "rsync is not installed"
  87. # ---- 1. the gates ---------------------------------------------------------------------------
  88. if [ "$SKIP" = 1 ]; then
  89. step "[1/5] GATES SKIPPED"
  90. printf ' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'
  91. printf ' !! --skip-tests: NOTHING WAS TESTED. You deploy untested code. !!\n'
  92. printf ' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'
  93. else
  94. step "[1/5] gates (${#GATES[@]})"
  95. # headless Chromes that exist BEFORE the gates are not ours (other sessions' test runs)
  96. chromes() { ps -eo pid=,ppid=,args= | awk '/[h]l-browser-tier/ && /remote-debugging-port/ && !/--type=/ {print $1, $2}'; }
  97. before=" $(chromes | awk '{print $1}' | tr '\n' ' ') "
  98. for g in "${GATES[@]}"; do
  99. printf ' $ %s\n' "$g"
  100. out=$(mktemp)
  101. if ! $g > "$out" 2>&1; then
  102. grep -E '^FAIL|passed,' "$out" | sed 's/^/ /' || true
  103. echo " (full output: $out)"
  104. die "gate failed: $g"
  105. fi
  106. grep -E 'passed,' "$out" | tail -1 | sed 's/^/ /' || true
  107. rm -f "$out"
  108. done
  109. # a Chrome our gates LEFT: new since the gates began AND no longer owned by a running node test
  110. leaked=""
  111. while read -r pid ppid; do
  112. [ -z "$pid" ] && continue
  113. case "$before" in *" $pid "*) continue ;; esac
  114. ps -o args= -p "$ppid" 2>/dev/null | grep -q '^node\|/node ' && continue
  115. leaked="$leaked $pid"
  116. done < <(chromes)
  117. if [ -n "$leaked" ]; then
  118. die "a gate left a headless Chrome (pids:$leaked; ps -eo pid,args | grep hl-browser-tier)"
  119. fi
  120. fi
  122. # ---- 2. pre-deploy backup of the LIVE data (before anything is sent) ------------------------
  123. BACKUP_ROOT=${DEPLOY_BACKUP_ROOT:-/media/SLOW1TB2/deploy-backups}
  124. BACKUP_DIR="$BACKUP_ROOT/$APP"
  125. BACKUP_FILE="$BACKUP_DIR/$APP-$(date +%Y%m%d-%H%M).tgz"
  126. if [ "$DRY" = 1 ]; then
  127. step "[2/5] DRY RUN: backup skipped"
  128. else
  129. step "[2/5] backup live data ($APP) → $BACKUP_FILE"
  130. mkdir -p "$BACKUP_DIR"
  131. if [ -n "$REMOTE_HOST" ]; then
  132. present=$($SSH "$REMOTE_HOST" "cd '$REMOTE_DIR' 2>/dev/null && for p in storage .sessions .env; do [ -e \"\$p\" ] && echo \"\$p\"; done; true")
  133. else
  134. present=$(cd "$REMOTE_DIR" 2>/dev/null && for p in storage .sessions .env; do [ -e "$p" ] && echo "$p"; done; true)
  135. fi
  136. present=$(printf '%s' "$present" | tr '\n' ' ' | sed 's/ *$//')
  137. if [ -z "$present" ]; then
  138. echo " nothing to back up yet (no storage/.sessions/.env on the target)"
  139. else
  140. echo " taring: $present"
  141. if [ -n "$REMOTE_HOST" ]; then
  142. # tar exit 1 = "file changed as we read it" (a background job writing, e.g. the photo backfill) — a warning, not a
  143. # failure; the archive is then checked with gzip -t. Anything else (ssh drop, tar exit 2) still refuses.
  144. run $SSH "$REMOTE_HOST" "tar czf - --warning=no-file-changed -C '$REMOTE_DIR' $present; r=\$?; [ \$r -le 1 ]" > "$BACKUP_FILE" \
  145. && gzip -t "$BACKUP_FILE" \
  146. || { rm -f "$BACKUP_FILE"; die "backup failed (tar/ssh error) — refusing to deploy"; }
  147. else
  148. run tar czf "$BACKUP_FILE" -C "$REMOTE_DIR" $present \
  149. || { rm -f "$BACKUP_FILE"; die "backup failed (tar error) — refusing to deploy"; }
  150. fi
  151. [ -s "$BACKUP_FILE" ] || { rm -f "$BACKUP_FILE"; die "backup is empty — refusing to deploy"; }
  152. echo " $(du -h "$BACKUP_FILE" | cut -f1) $BACKUP_FILE"
  153. ls -1t "$BACKUP_DIR/$APP"-*.tgz 2>/dev/null | tail -n +6 | xargs -r rm -f --
  154. echo " keeping $(ls -1 "$BACKUP_DIR/$APP"-*.tgz 2>/dev/null | wc -l) archive(s) of $APP"
  155. fi
  156. fi
  158. # ---- 3. rsync the code ----------------------------------------------------------------------
  159. RSYNC=(rsync -az --no-owner --no-group --itemize-changes)
  160. for e in "${EXCLUDES[@]}"; do RSYNC+=("--exclude=$e"); done
  161. if [ -n "$REMOTE_HOST" ]; then
  162. RSYNC+=(-e "$SSH")
  163. else
  164. mkdir -p "$REMOTE_DIR"
  165. fi
  166. # the preview is ALWAYS made first (rsync -n): nothing that must stay on Byrodin may be in it
  167. step "[3/5] rsync preview (what would be sent)"
  168. preview=$(mktemp)
  169. run "${RSYNC[@]}" -n ./ "$TARGET/" > "$preview"
  170. sed 's/^/ /' "$preview"
  171. echo " ($(grep -c . "$preview" || true) lines)"
  172. if awk '{print $2}' "$preview" | grep -E '^(storage/|\.sessions/|\.env|\.scratch/|server\.)|\.log$|\.pid$' ; then
  173. die "the preview holds a path that must never be sent (see above)"
  174. fi
  175. rm -f "$preview"
  176. if [ "$DRY" = 1 ]; then
  177. step "[3/5] DRY RUN: nothing sent"
  178. else
  179. step "[3/5] rsync"
  180. run "${RSYNC[@]}" ./ "$TARGET/" | sed 's/^/ /'
  181. fi
  183. # ---- 4. restart the container ---------------------------------------------------------------
  184. RESTART="cd '$REMOTE_DIR' && docker compose up -d --build && docker compose restart && docker compose ps"
  185. if [ "$DRY" = 1 ]; then
  186. step "[4/5] DRY RUN: would restart $CONTAINER"
  187. if [ -n "$REMOTE_HOST" ]; then echo " would run: $SSH $REMOTE_HOST \"$RESTART\""; else echo " would run: bash -c \"$RESTART\""; fi
  188. else
  189. step "[4/5] restart $CONTAINER"
  190. if [ -n "$REMOTE_HOST" ]; then
  191. run $SSH "$REMOTE_HOST" "$RESTART"
  192. else
  193. run bash -c "$RESTART"
  194. fi
  195. fi
  197. # ---- 5. the URL answers 200 -----------------------------------------------------------------
  198. if [ "$DRY" = 1 ]; then
  199. step "[5/5] DRY RUN: would check $URL answers 200"
  200. step "dry run done — nothing was sent, nothing restarted"
  201. exit 0
  202. fi
  203. # tracker#31 (mission 026): the boot is slower now (the search index over ~100k people, the list caches, the slug map) — nginx
  204. # answers 502 until the app listens. Ask every 2 s for up to URL_WAIT seconds (default 90), print only changes of the answer.
  205. URL_WAIT=${DEPLOY_URL_WAIT:-90}
  206. step "[5/5] $URL must answer 200 (waiting up to ${URL_WAIT} s for the boot)"
  207. code=000
  208. last=
  209. start=$(date +%s)
  210. while :; do
  211. code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$URL" || true)
  212. waited=$(( $(date +%s) - start ))
  213. if [ "$code" != "$last" ] || [ "$code" = 200 ]; then echo " after ${waited} s: $code"; last=$code; fi
  214. [ "$code" = 200 ] && break
  215. [ "$waited" -ge "$URL_WAIT" ] && break
  216. sleep 2
  217. done
  218. if [ "$code" != 200 ]; then
  219. if [ -n "$REMOTE_HOST" ]; then die "$URL answered $code, not 200, after ${URL_WAIT} s — look: $SSH $REMOTE_HOST docker logs --tail 50 $CONTAINER"; fi
  220. die "$URL answered $code, not 200, after ${URL_WAIT} s — look: docker logs --tail 50 $CONTAINER"
  221. fi
  222. step "deployed $APP → $TARGET, $URL answers 200"

Branches

Latest commits

  • 5cb85d75deploy.sh: a backup taken while a background job writes (tar exit 1) is a warning; archive checked with gzip -tmre
  • 9abda75dtracker: report 035mre
  • 12595e47mission 035: theme re-vendored from layouts.worldapi.org 0222f67 (two corner radii: radiusSmall 5px, radiusLarge 10px); the tracker's 18 own radii -> radiusSmall/radiusLarge (--layout-radius is gone); check-theme 0; gates 379/0, 32/0, 53/0, 229/0, 26/0; real copy: every computed radius in {0, 5px, 10px, 50%}mre
  • e7305014tracker: report 032 (art + photos)mre
  • fbb903cctracker#33/#37 (mission 032): a title without a TMDB poster gets its backdrop (w780, posterFromBackdrop, shown 2:3 centre-cropped); movies store runtime, the page shows release date + runtime; art backfill (public titles without poster file / movies without runtime) and person photo backfill (tmdbProfile / photoCheck) as the last start jobs (TRACKER_ART, TRACKER_PHOTOS; off in every gate start); gates 379/0, 32/0, 53/0, 229/0, 26/0, check-theme 0; live copy: art 4977 titles in 42 min (115 posters, 15 backdrops, 4609 runtimes), The Remaining shows its backdrop + 7 minmre
  • 4ecc67b2tracker: STATUS/LOG for the t38 + t40 merge (gates 374/0, 32/0, 53/0, 229/0, 26/0, check-theme 0; live copy checks)mre
  • e5945d2fMerge t40 (tracker#40 curated franchises, Franchises menu, superseded collections) into main: deploy.sh lists all six gates (browser, kinds, franchises, pager, franchiseseed, check-theme); search.hl keeps #37's personPhotoOf + #40's importWithCredits; pager gate runs with TRACKER_FRANCHISE_SEED=0; gates 374/0, 32/0, 53/0, 229/0, 26/0, check-theme 0mre
  • b638e99dMerge t38 (tracker#38 pagination, #35 Returning/Airing label) into main: LOG/STATUS keep both sides; pager gate follows #37's /people (everyone, last updated first: seed updatedAt); gates pager 229/0, browser 374/0mre
  • f8ffa4dbtracker: report 032 + The Remaining + #39mre
  • 7565a863tracker: LOG timemre
  • b10f00c8tracker#39: double episodes — migrated episodes whose TMDB id TMDB replaced are adopted by their number in the sync (old id -> migratedTmdbId); merge.hl step 3 merges each season's doubles at start (keeper: most watches > synced > first; watches moved/parked; tombstones into mergedEpisodes, nothing deleted); tools/count-duplicate-episodes.hl; gate fixture + paths-m039; live copy 850 -> 0 in 64 s; gates 373/0, 32/0, 52/0mre
  • 8f1d4542tracker#40: superseded collections — a TMDB collection timeline whose titles are all in one curated timeline is hidden (supersededBy; kept: own page + editor finder), set by the collection seed when it makes one and by the curated build (lifted when the cover is gone); partly covered ones join that franchise; no second widget (First Contact: only Star Trek — Prime); gate franchiseseed 26/0, browser 365/0, kinds 32/0, franchises 53/0, check-theme 0; real copy 24 supersededmre
  • 9448d643tracker#35 follow-up (mission 033): TVmaze 'Running' is labelled 'Returning', or 'Airing' while a non-special episode of the two newest seasons is released within today +-7 days (data unchanged); gate fixtures Running/Airing/Aired + a special; browser 366/0, kinds 32/0, franchises 52/0, pager 229/0, check-theme 0mre
  • 7d4b293dtracker#40: "Franchises" in the main menu (desktop header after People, phone sidebar) → /franchises, marked on franchise and timeline pages; gates 365/0, 32/0, 53/0, 24/0, check-theme 0mre
  • 8751adb8tracker: report 032mre
  • 9bce1f65tracker mission 032: STATUS gate files + the hour-boundary flakemre
  • 718bfb89tracker#37 (mission 032): /people = everyone, last updated first (updatedAt stamped by the person fill; view built at boot, touched people first at once), photo + name tiles (person colour) with the /movies pagination, /people/<letter> removed; photo = our file, tmdbProfile, a cast/crew entry's profile (in-memory map at boot), else the new 'no photo' placeholder; new cast/crew/created_by people keep tmdbProfile; search people rows with the photo; /settings = the heading only; util.hl sortDesc starts from sorted runs (same result, 105k: 1.6 s -> 0.15 s); gates 369/0, 32/0, 52/0, check-theme 0; README/STATUS/LOGmre
  • 93dfb0batracker#40 (mission 034): the curated franchises — data/franchises.json (17 franchises, 31 timelines, 285 TMDB titles, movies + series, in-universe/release order, 12 TMDB collections attached); lib/franchiseseed.hl + jobs.hl franchiseSeedTick (last start job, imports missing titles via details.hl importWithCredits = the search's Add, one per step paced, then one build; franchiseseed.db: editor changes win, the creator's same-name franchise adopted / timeline left alone, 404 remembered, resumable, idempotent); timeline heads 'N titles · in-universe order' (orderKind) and wrap on a phone; series pages show the widget; new gate tests/franchiseseed.mjs (5th in deploy.sh), the others run with TRACKER_FRANCHISE_SEED=0; gates 365/0, 32/0, 52/0, 24/0, check-theme 0; real copy 196 imported, 0 failed, 7 min, restart unchanged=31mre
  • 03ec792ftracker#38 (mission 033): pagination goes exactly to the clicked page — tilelist read the clicked button's text after pagination.hl's own handler had rebuilt the buttons (real clicks only); now li.current, else the button's own text; new gate tests/pager.mjs (5 lists x 11 pages, 390/1280, real + script clicks, Back/Forward) 229/0; browser 365/0, kinds 32/0, franchises 52/0, check-theme 0mre
  • 96ba683adeploy.sh: a gate without a 'passed,' line (check-theme) no longer ends the scriptmre