tracker

30.0 KB

1. // project.hl — tracker.worldapi.org: STEP 1 (tracker.worldapi.org#1), an empty shell. Login copied unchanged from
2. // calendar.worldapi.org (rejected once for a centered sign-in and no header selector — architect, 2026-09-27):
3. // ident only, no own passwords (README "How apps use ident" of ident.worldapi.org), the identity selector in the
4. // header (components/main.hl). No shows, no data yet — later steps come from the creator (CONCEPT.md).
5. //
6. // the header's ident-selector / "Log in with ident" -> <ident>/login?key=&return=<this app>/login/callback
7. // /login/callback?ident_code=<code> the app's server exchanges the code for the identity's short id
8. // (POST <ident>/api/exchange) and signs this app's OWN session in
9. // (session.user = { id = <users @id> }, users.hl)
10. // the header's "Log out" button signs this app's session out via the face trackerLogOut (components/main.hl)
11. // — ident's own session, and the browser's ident cookie, are untouched
12. //
13. // Config (env; no committed secret — the first deploy registers this app in ident and sets these, done by the
14. // architect, as with ident's own .env for SMTP):
15. // TRACKER_PORT (45008), TRACKER_URL (this app's own public address, for the return URL),
16. // IDENT_URL (https://ident.worldapi.org), TRACKER_KEY (pk_…), TRACKER_SECRET (sk_…)
17. import WebFramework from 'hl:web'
18. import { Response } from 'hl:http1'
19. import { randomBytes } from 'hl:crypto'
20. import { env } from 'hl:proc'
21. import { readBytes, readFile, exists } from 'hl:fs'
22. import { now, timestamp, every } from 'hl:time'
24. import Styles from './styles.hl'
25. import { exchangeCode, ensureUser } from './users.hl'
26. import Home from './components/home.hl'
27. import LoginFailed from './components/loginfailed.hl'
28. import Show from './components/show.hl'
29. import Unwatched from './components/unwatched.hl'
30. import Schedule from './components/schedule.hl'
31. import MyShows from './components/myshows.hl'
32. import MyMovies from './components/mymovies.hl'
33. import Movies from './components/movies.hl'
34. import AllShows from './components/allshows.hl'
35. import Search from './components/search.hl'
36. import Person from './components/person.hl'
37. import { ensureIndex, searchTitleChanged } from './search.hl'
38. import { syncEnabled, syncShow, persistSync, emptyTotals, addTotals, totalsLine, pauseMsAfter, msPerRequest, adultBackfillQueue, backfillTitle } from './tmdbsync.hl'
39. import { detailsRepairQueue, completeStep, completeLine, persistDetails, persistCredits, applyCredits, creditsQueue, creditsStep } from './details.hl'
40. import { allFollowedShowIds } from './follows.hl'
41. import { refreshCatalogShow } from './catalog.hl'
42. import { dark, darker } from './shared/tokens.hl'
44. static appTitle = "tracker"
46. // ---- THE INSTALLABLE APP (tracker.worldapi.org#10), the same way calendar.worldapi.org does it: hl:web's own web app
47. // manifest (/__hl/manifest.webmanifest, linked from every head with the apple-touch-icon and theme-color) and service
48. // worker (/__hl/sw.js) from these settings — no JavaScript of ours. Icons in icons/ (icon.svg is the source, the PNGs
49. // are rendered from it with rsvg-convert, README "Icons"). The theme colour is the header's background (darker), the
50. // splash background the page's (dark) — both off the tokens, not written out again.
51. appThemeColor = darker.value
52. appBackgroundColor = dark.value
53. appIcons = [
54. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'any' }
55. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'any' }
56. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'maskable' }
57. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'maskable' }
58. ]
59. appTouchIcon = '/icons/apple-touch-icon.png'
60. appFavicon = '/icons/favicon.svg'
61. // OFFLINE: only the shell — the header and `/` (components/main.hl says "You are offline" while the browser has no
62. // network). The data pages are not kept: without a network they get hl:web's "Unavailable offline" page.
63. offline = [ Home ]
65. styles = Styles
67. port = env('TRACKER_PORT') != null ? toNumber(env('TRACKER_PORT')) : 45008
68. watching = env('TRACKER_WATCH') != '0'
69. // read by hl:web's own manifest config (WebFramework.hl `cfg.sessionDir`), not the constructor call below
70. sessionDir = env('TRACKER_SESSIONS') != null ? env('TRACKER_SESSIONS') : null
72. // ---- THE LOGIN BUTTON'S RETURN (ident README "How apps use ident"), copied from calendar.worldapi.org's project.hl
73. // BACK TO THE PAGE: /login.js puts `?next=` into the button's return URL at the click. Only a same-origin PATH
74. // goes (one `/`, URL-safe characters, ≤ 500). Anything else → `/`.
75. nextChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~/?&=%+,;@!$()*:'
76. safePath = (want) => {
77. if (want == null || hlTypeName(want) != 'String' || want == '' || want.length > 500) { return '/' }
78. if (want.slice(0, 1) != '/' || want.slice(0, 2) == '//' || want.slice(0, 7) == '/login/') { return '/' }
79. let i = 0
80. while (i < want.length) {
81. if (!nextChars.includes(want[i])) { return '/' }
82. i = i + 1
83. }
84. return want
85. }
87. // A FAILED LOGIN is a page (components/loginfailed.hl): the reason is parked in the session, then → /login/failed
88. failed = (req, why) => {
89. let s = req.session
90. let fresh = s == null
91. if (fresh) { s = server.sessions.mint() }
92. s.data.loginError = why
93. server.sessions.save(s)
94. let res = new Response('login failed: ' + why, { status = 302 headers = { 'Location' = '/login/failed' 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
95. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
96. return res
97. }
99. // the function route gets the cookie's session as req.session (hybriel #11); none yet → minted here
100. loginCallback = (route, req) => {
101. if (req.method != 'GET') { return failed(req, 'GET only') }
102. let q = req.query != null ? req.query : {}
103. let code = q.ident_code
104. if (code == null || code == '') { return failed(req, 'ident sent no login code') }
105. let x = exchangeCode(code)
106. if (x.error != null) { return failed(req, x.error) }
107. let u = ensureUser(x.identity)
108. if (u == null) { return failed(req, 'could not store the user') }
109. let s = req.session
110. let fresh = s == null
111. if (fresh) { s = server.sessions.mint() }
112. s.user = { id = u.id }
113. s.data.tag = randomBytes(16)
114. s.data.loginError = null
115. server.sessions.save(s)
116. let res = new Response('logged in', { status = 302 headers = { 'Location' = safePath(q.next) 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
117. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
118. return res
119. }
121. // ---- /posters/<name> (tracker.worldapi.org#4): a show's poster image, served dynamically from
122. // storage/mpackdb/posters/ (the poster's file name is the show's oldId + its migrated extension,
123. // shows.hl posterName) — falls back to a placeholder svg when the real file isn't there yet (the
124. // mongo export never included the actual poster bytes, see the report's `open`).
125. postersDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/posters'
126. placeholderPoster = './assets/poster-placeholder.svg'
127. posterNameChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_'
128. isSafePosterName = (name) => {
129. if (name == null || hlTypeName(name) != 'String' || name == '' || name.length > 100) { return false }
130. let i = 0
131. while (i < name.length) {
132. if (!posterNameChars.includes(name[i])) { return false }
133. i = i + 1
134. }
135. return true
136. }
137. endsWithStr = (s, suffix) => { return s.length >= suffix.length && s.slice(s.length - suffix.length) == suffix }
138. mimeOfPoster = (name) => {
139. if (endsWithStr(name, '.png')) { return 'image/png' }
140. if (endsWithStr(name, '.webp')) { return 'image/webp' }
141. if (endsWithStr(name, '.svg')) { return 'image/svg+xml' }
142. return 'image/jpeg'
143. }
144. posterRoute = (route, req) => {
145. let name = route.params != null ? route.params.name : null
146. if (!isSafePosterName(name)) { return new Response('bad poster name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
147. let path = postersDir + '/' + name
148. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
149. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
150. }
152. // ---- /profiles/<name> (tracker.worldapi.org#16): a person's photo, from storage/mpackdb/profiles/ (people.hl syncProfile
153. // downloads it on the first visit of the person page) — the same checks and placeholder as /posters/<name>
154. profilesDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/profiles'
155. profileRoute = (route, req) => {
156. let name = route.params != null ? route.params.name : null
157. if (!isSafePosterName(name)) { return new Response('bad photo name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
158. let path = profilesDir + '/' + name
159. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
160. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
161. }
163. // ---- the old addresses of the personal lists (tracker.worldapi.org#8): they all live under /my/ now —
164. // a permanent redirect so old links keep working (architect's decision)
165. movedTo = (path) => { return new Response('moved to ' + path, { status = 301 headers = { 'Location' = path 'Content-Type' = 'text/plain; charset=utf-8' } }) }
166. unwatchedMoved = (route, req) => { return movedTo('/my/unwatched') }
167. scheduleMoved = (route, req) => { return movedTo('/my/schedule') }
169. routes = [
170. { pattern = "/favicon.ico" file = "./icons/favicon.ico" headers = { 'Cache-Control' = 'no-cache' } }
171. // the installable app (tracker.worldapi.org#10): the icons (manifest and service worker are hl:web's own, from appIcons / offline)
172. { pattern = "/icons/icon-192.png" file = "./icons/icon-192.png" headers = { 'Cache-Control' = 'no-cache' } }
173. { pattern = "/icons/icon-512.png" file = "./icons/icon-512.png" headers = { 'Cache-Control' = 'no-cache' } }
174. { pattern = "/icons/apple-touch-icon.png" file = "./icons/apple-touch-icon.png" headers = { 'Cache-Control' = 'no-cache' } }
175. { pattern = "/icons/favicon.svg" file = "./icons/favicon.svg" headers = { 'Cache-Control' = 'no-cache' } }
176. { pattern = "/login/callback" function = loginCallback }
177. { pattern = "/login/failed" component = LoginFailed }
178. { pattern = "/login.js" file = "./login.js" headers = { 'Cache-Control' = 'no-cache' } }
179. { pattern = "/posters/:name" function = posterRoute }
180. // tracker.worldapi.org#13: every movie / every series, 24 per page — the page is a PATH segment (`/movies/page/2`), a
181. // page component cannot read `?page=` (hybriel#11) — and the movies the user follows
182. { pattern = "/movies" component = Movies }
183. { pattern = "/movies/page/:page" component = Movies }
184. { pattern = "/shows" component = AllShows }
185. { pattern = "/shows/page/:page" component = AllShows }
186. { pattern = "/shows/:slug" component = Show }
187. { pattern = "/my/shows" component = MyShows }
188. { pattern = "/my/movies" component = MyMovies }
189. { pattern = "/my/unwatched" component = Unwatched }
190. { pattern = "/my/schedule" component = Schedule }
191. { pattern = "/unwatched" function = unwatchedMoved }
192. { pattern = "/schedule" function = scheduleMoved }
193. // tracker.worldapi.org#14: the search — `/search/<text>` (hl:web gives a page no query string, so not `?q=`)
194. { pattern = "/search/:q?" component = Search }
195. // tracker.worldapi.org#16: a person's page (the show page's cast and the search link here) and their photo
196. { pattern = "/person/:slug" component = Person }
197. { pattern = "/profiles/:name" function = profileRoute }
198. { pattern = "/" component = Home }
199. ]
201. // WHO GETS THE PUSH: the login state reaches the tabs of one session (components/main.hl trackerSignedIn/Out).
202. tagOf = (session) => { return session != null && session.data != null ? session.data.tag : null }
203. audience = {
204. trackerSignedIn = (tag, session) => { return tag != null && tagOf(session) == tag }
205. trackerSignedOut = (tag, session) => { return tag != null && tagOf(session) == tag }
206. }
208. // cookies are per host, not per port: an own name keeps this app's session apart from
209. // ident's own (and from any other worldapi app sharing a dev host), see ident README "Design
210. // tokens" / hybriel#10 hybriel#17.
211. sessionCookie = 'trackersid'
212. // tracker.worldapi.org#14: the search index (search.hl) is built once here at boot (~0.6 s on the real data), not by the
213. // first search
214. searchIndexReady = ensureIndex()
215. server = new WebFramework(routes = routes, styles = styles, port = port, sessionCookie = sessionCookie, watchMode = watching)
217. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, tmdbsync.hl) --------------------------------------
218. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC) every
219. // followed show is synced from TMDB. hl:web serves ONE request at a time, so the run never takes the server
220. // for itself: a 1-second clock does ONE show per tick (1–3 TMDB requests + maybe its poster) and returns — the
221. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
222. // After a show that made n requests the next waits n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze lookup
223. // (≤ 20 / 10 s, tracker.worldapi.org#12).
224. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
225. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
226. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
227. syncQueue = []
228. syncAt = 0
229. syncDay = ''
230. syncNextAt = 0
231. syncTotals = emptyTotals()
232. syncTick = () => {
233. let t = now()
234. if (syncQueue.length == 0) {
235. let stamp = timestamp(t)
236. let day = stamp.slice(0, 10)
237. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
238. syncDay = day
239. syncQueue = allFollowedShowIds()
240. syncAt = t
241. syncNextAt = 0
242. syncTotals = emptyTotals()
243. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows')
244. return null
245. }
246. if (t < syncNextAt) { return null }
247. let id = syncQueue[0]
248. let rest = []
249. let i = 1
250. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
251. syncQueue = rest
252. let r = syncShow(id)
253. // tracker#28: the cast + crew came with the details request (details.hl applyCredits)
254. if (r.details != null) {
255. let c = applyCredits(id, r.details)
256. if (c.error != '') { console.log('tmdb sync: ' + c.error) }
257. }
258. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
259. // also brings TMDB's adult flag (mission 054) → the search's view of it too
260. refreshCatalogShow(id)
261. searchTitleChanged(id)
262. let took = now() - t
263. syncTotals = addTotals(syncTotals, r)
264. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
265. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
266. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
267. syncNextAt = now() + pauseMsAfter(r)
268. if (syncQueue.length == 0 || syncTotals.shows % 10 == 0) { persistDetails() }
269. if (syncQueue.length == 0) { console.log('tmdb sync done: ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000)) }
270. return null
271. }
273. // ---- THE ADULT-FLAG BACKFILL (mission 054, tmdbsync.hl backfillTitle) -----------------------------------
274. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
275. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
276. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
277. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
278. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
279. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
280. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
281. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
282. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
283. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
284. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
285. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
286. backfillQueue = null
287. backfillPos = 0
288. backfillDone = false
289. backfillAt = 0
290. backfillNextAt = 0
291. backfillTries = {}
292. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
293. backfillLine = () => {
294. let b = backfillTotals
295. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
296. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
297. }
298. backfillTick = () => {
299. if (backfillDone) { return null }
300. let t = now()
301. if (backfillQueue == null) {
302. backfillQueue = adultBackfillQueue()
303. backfillAt = t
304. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
305. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
306. return null
307. }
308. if (t < backfillNextAt) { return null }
309. let id = backfillQueue[backfillPos]
310. backfillPos = backfillPos + 1
311. let r = backfillTitle(id)
312. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
313. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
314. let q = backfillQueue
315. q.push(id)
316. backfillQueue = q
317. backfillTotals.retries = backfillTotals.retries + 1
318. } else {
319. backfillTotals.titles = backfillTotals.titles + 1
320. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
321. }
322. backfillTotals.posters = backfillTotals.posters + r.posters
323. backfillTotals.ids = backfillTotals.ids + r.ids
324. backfillTotals.requests = backfillTotals.requests + r.requests
325. backfillTotals.errors = backfillTotals.errors + r.errors
326. if (r.error != '') { console.log('adult backfill: ' + r.error) }
327. if (r.adult != null) {
328. refreshCatalogShow(id)
329. searchTitleChanged(id)
330. }
331. let took = now() - t
332. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
333. backfillNextAt = now() + r.requests * msPerRequest
334. let finished = backfillPos >= backfillQueue.length
335. if (finished || backfillPos % 25 == 0) { persistSync() }
336. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
337. if (finished) {
338. backfillDone = true
339. console.log('adult backfill done: ' + backfillLine())
340. }
341. return null
342. }
344. // ---- THE DETAILS REPAIR (tracker.worldapi.org#26, details.hl completeStep) ----------------------------------------
345. // Titles the person pages added from a filmography (#16) are MINIMAL (no poster, seasons, ids, cast …). Opening one completes
346. // it (components/show.hl); this job completes the rest in the background: at start it collects every incomplete, non-adult
347. // title and does ONE per step on the app's clock, like the adult backfill — a step is the daily sync's work for one title
348. // (1–3 TMDB requests + poster + TVmaze), requests queued meanwhile are served between two steps; then a pause of 260 ms per
349. // TMDB request + 550 ms per TVmaze request (pauseMsAfter: ≤ 40 / 10 s, ≤ 20 / 10 s). RESUMES after a restart (the queue is
350. // what is still incomplete); TMDB 404 marks the title (`detailsCheck`), a passing failure goes to the back of the queue (at
351. // most 2 more tries per start). Waits while the daily sync or the adult backfill runs. Persisted every 10 titles + at the end.
352. // Log: start, every TRACKER_REPAIR_LOG_EVERY (100) titles, each failure, done. TRACKER_REPAIR=0 switches it off (off too
353. // without TMDB_READ_TOKEN). TRACKER_REPAIR_FAST=1 (the gate, fake TMDB only): 20 ms pause instead.
354. repairOn = env('TRACKER_REPAIR') != '0' && syncEnabled
355. repairFast = env('TRACKER_REPAIR_FAST') == '1'
356. repairLogEvery = env('TRACKER_REPAIR_LOG_EVERY') != null ? toNumber(env('TRACKER_REPAIR_LOG_EVERY')) : 100
357. repairQueue = null
358. repairPos = 0
359. repairDone = false
360. repairAt = 0
361. repairNextAt = 0
362. repairTries = {}
363. repairTotals = { titles = 0 complete = 0 failed = 0 adult = 0 newSeasons = 0 newEpisodes = 0 posters = 0 cast = 0 requests = 0 tvmazeRequests = 0 retries = 0 }
364. repairLine = () => {
365. let b = repairTotals
366. return b.titles + '/' + repairQueue.length + ' titles, complete=' + b.complete + ' failed=' + b.failed + ' adult=' + b.adult + ' newSeasons=' + b.newSeasons + ' newEpisodes=' + b.newEpisodes
367. + ' posters=' + b.posters + ' cast=' + b.cast + ' requests=' + b.requests + ' tvmazeRequests=' + b.tvmazeRequests + ' retries=' + b.retries + ' seconds=' + ((now() - repairAt) / 1000)
368. }
369. repairTick = () => {
370. if (repairDone) { return null }
371. let t = now()
372. if (repairQueue == null) {
373. repairQueue = detailsRepairQueue()
374. repairAt = t
375. console.log('details repair: start, ' + repairQueue.length + ' incomplete titles (' + (now() - t) + ' ms to find them)')
376. if (repairQueue.length == 0) { repairDone = true console.log('details repair: nothing to do') }
377. return null
378. }
379. if (t < repairNextAt) { return null }
380. let id = repairQueue[repairPos]
381. let r = completeStep(id)
382. let took = now() - t
383. if (took > 1500) { console.log('details repair: slow step, show ' + id + ', ' + took + ' ms') }
384. if (r.error != '') { console.log('details repair: ' + r.error) }
385. repairNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
386. // a series takes several steps: the same title again next time
387. if (!r.done && !r.retry) { return null }
388. repairPos = repairPos + 1
389. if (r.retry && (repairTries[id] == null || repairTries[id] < 2)) {
390. repairTries[id] = repairTries[id] == null ? 1 : repairTries[id] + 1
391. let q = repairQueue
392. q.push(id)
393. repairQueue = q
394. repairTotals.retries = repairTotals.retries + 1
395. } else if (!r.skipped) {
396. // done: its counts (a failed first step: the request alone)
397. let sum = r.sum
398. repairTotals.titles = repairTotals.titles + 1
399. if (r.adult == true) { repairTotals.adult = repairTotals.adult + 1 } else if (sum.steps == 0) { repairTotals.failed = repairTotals.failed + 1 } else { repairTotals.complete = repairTotals.complete + 1 }
400. repairTotals.newSeasons = repairTotals.newSeasons + sum.newSeasons
401. repairTotals.newEpisodes = repairTotals.newEpisodes + sum.newEpisodes
402. repairTotals.posters = repairTotals.posters + sum.posters
403. repairTotals.cast = repairTotals.cast + sum.cast
404. repairTotals.requests = repairTotals.requests + (sum.steps == 0 ? r.requests : sum.requests)
405. repairTotals.tvmazeRequests = repairTotals.tvmazeRequests + sum.tvmazeRequests
406. }
407. let finished = repairPos >= repairQueue.length
408. if (finished || repairPos % 10 == 0) { persistDetails() }
409. if (!finished && !r.skipped && !r.retry && repairTotals.titles > 0 && repairTotals.titles % repairLogEvery == 0) { console.log('details repair: ' + repairLine()) }
410. if (finished) {
411. repairDone = true
412. console.log('details repair done: ' + repairLine())
413. }
414. return null
415. }
417. // ---- THE CREDITS JOB (tracker.worldapi.org#28, details.hl creditsStep) ------------------------------------------------------
418. // Every PUBLIC title (adult == false; adult/unknown are skipped) with a TMDB id gets its whole cast + crew once: at start (after
419. // the adult backfill and the details repair are done) the job collects every such title without `creditsAt` and asks ONE per
420. // step on the app's clock — one request (details + credits / aggregate_credits), then 260 ms (≤ 40 / 10 s); requests queued
421. // meanwhile are served between two steps. Titles the details repair, the daily sync or an import already did carry `creditsAt`
422. // and are not asked. RESUMES after a restart (the queue = what still has no credits); TMDB 404 / adult → `creditsCheck`, never
423. // asked again; a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily sync
424. // runs. Persisted (shows + people only) every 25 titles + at the end. Log: start, every TRACKER_CREDITS_LOG_EVERY (500) titles, failures, slow steps
425. // (> 1.5 s), done. TRACKER_CREDITS=0 switches it off (off too without TMDB_READ_TOKEN). TRACKER_CREDITS_FAST=1 (the gate, fake
426. // TMDB only): 20 ms pause instead.
427. // MEMORY LIMIT: the runtime never frees what a fetch + JSON parse took (mission 061 measured ~6 MB more RSS per title on the real
428. // data, 300 fetches of a 440 KB JSON = +2.7 GB in a 10-line script) — the whole job in one process would add ~35 GB. Before every
429. // step the job reads its own RSS (/proc/self/status); above TRACKER_CREDITS_MAX_MB (16000) it PAUSES for this process (logged) and
430. // goes on with what is left after the next restart (deploy, `docker restart`).
431. creditsOn = env('TRACKER_CREDITS') != '0' && syncEnabled
432. creditsFast = env('TRACKER_CREDITS_FAST') == '1'
433. creditsLogEvery = env('TRACKER_CREDITS_LOG_EVERY') != null ? toNumber(env('TRACKER_CREDITS_LOG_EVERY')) : 500
434. creditsQueueIds = null
435. creditsPos = 0
436. creditsDone = false
437. creditsAt = 0
438. creditsNextAt = 0
439. creditsTries = {}
440. creditsMaxMb = env('TRACKER_CREDITS_MAX_MB') != null ? toNumber(env('TRACKER_CREDITS_MAX_MB')) : 16000
441. creditsTotals = { titles = 0 done = 0 failed = 0 cast = 0 crew = 0 people = 0 requests = 0 retries = 0 slowest = 0 }
442. // this process' resident memory in MB (VmRSS), 0 when unknown
443. rssMb = () => {
444. if (!exists('/proc/self/status')) { return 0 }
445. let t = readFile('/proc/self/status')
446. let i = t != null ? t.indexOf('VmRSS:') : -1
447. if (i < 0) { return 0 }
448. let kb = toNumber(t.slice(i + 6, t.indexOf('kB', i)).trim())
449. return kb != null ? (kb - kb % 1024) / 1024 : 0
450. }
451. creditsLine = () => {
452. let b = creditsTotals
453. return b.titles + '/' + creditsQueueIds.length + ' titles, done=' + b.done + ' failed=' + b.failed + ' cast=' + b.cast + ' crew=' + b.crew + ' people=' + b.people
454. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - creditsAt) / 1000)
455. }
456. creditsTick = () => {
457. if (creditsDone) { return null }
458. let t = now()
459. if (creditsQueueIds == null) {
460. creditsQueueIds = creditsQueue()
461. creditsAt = t
462. console.log('credits job: start, ' + creditsQueueIds.length + ' titles without credits (' + (now() - t) + ' ms to find them)')
463. if (creditsQueueIds.length == 0) { creditsDone = true console.log('credits job: nothing to do') }
464. return null
465. }
466. if (t < creditsNextAt) { return null }
467. let mb = rssMb()
468. if (mb > creditsMaxMb) {
469. creditsDone = true
470. persistCredits()
471. console.log('credits job: paused, RSS ' + mb + ' MB > TRACKER_CREDITS_MAX_MB ' + creditsMaxMb + ' — ' + (creditsQueueIds.length - creditsPos) + ' titles left for the next start; ' + creditsLine())
472. return null
473. }
474. let id = creditsQueueIds[creditsPos]
475. creditsPos = creditsPos + 1
476. let r = creditsStep(id)
477. let took = now() - t
478. if (took > creditsTotals.slowest) { creditsTotals.slowest = took }
479. if (took > 1500) { console.log('credits job: slow step, show ' + id + ', ' + took + ' ms, cast=' + r.cast + ' crew=' + r.crew + ' people=' + r.people) }
480. if (r.error != '') { console.log('credits job: ' + r.error) }
481. creditsNextAt = now() + (creditsFast ? 20 : r.requests * msPerRequest)
482. if (r.retry && (creditsTries[id] == null || creditsTries[id] < 2)) {
483. creditsTries[id] = creditsTries[id] == null ? 1 : creditsTries[id] + 1
484. let q = creditsQueueIds
485. q.push(id)
486. creditsQueueIds = q
487. creditsTotals.retries = creditsTotals.retries + 1
488. } else if (!r.skipped) {
489. creditsTotals.titles = creditsTotals.titles + 1
490. if (r.error != '') { creditsTotals.failed = creditsTotals.failed + 1 } else { creditsTotals.done = creditsTotals.done + 1 }
491. creditsTotals.cast = creditsTotals.cast + r.cast
492. creditsTotals.crew = creditsTotals.crew + r.crew
493. creditsTotals.people = creditsTotals.people + r.people
494. }
495. creditsTotals.requests = creditsTotals.requests + r.requests
496. let finished = creditsPos >= creditsQueueIds.length
497. if (finished || creditsPos % 25 == 0) { persistCredits() }
498. if (!finished && !r.skipped && !r.retry && creditsTotals.titles > 0 && creditsTotals.titles % creditsLogEvery == 0) { console.log('credits job: ' + creditsLine()) }
499. if (finished) {
500. creditsDone = true
501. console.log('credits job done: ' + creditsLine())
502. }
503. return null
504. }
506. // ONE clock for all four (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in
507. // between, only while no daily run is going on; the details repair only when neither has work; the credits job (#28) last
508. clockOn = syncOn || backfillOn || repairOn || creditsOn
509. clockTicks = 0
510. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
511. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
512. if (!repairOn) { console.log('details repair: off (' + (syncEnabled ? 'TRACKER_REPAIR=0' : 'no TMDB_READ_TOKEN') + ')') }
513. if (!creditsOn) { console.log('credits job: off (' + (syncEnabled ? 'TRACKER_CREDITS=0' : 'no TMDB_READ_TOKEN') + ')') }
514. if (clockOn) {
515. syncClock = every(0.1)
516. on syncClock.tick(x) {
517. clockTicks = clockTicks + 1
518. let busy = syncQueue.length > 0
519. if (syncOn && clockTicks % 10 == 0) { syncTick() }
520. let free = !busy && syncQueue.length == 0
521. if (backfillOn && free && !backfillDone) { backfillTick() }
522. else if (repairOn && free && (!backfillOn || backfillDone) && !repairDone) { repairTick() }
523. else if (creditsOn && free && (!backfillOn || backfillDone) && (!repairOn || repairDone)) { creditsTick() }
524. return null
525. }
526. }
528. on Error(e) { console.log('error absorbed: ' + e.message) }