tracker

48.9 KB

1. // project.hl — tracker.worldapi.org: STEP 1 (tracker.worldapi.org#1), an empty shell. Login copied unchanged from
2. // calendar.worldapi.org (rejected once for a centered sign-in and no header selector — architect, 2026-09-27):
3. // ident only, no own passwords (README "How apps use ident" of ident.worldapi.org), the identity selector in the
4. // header (components/main.hl). No shows, no data yet — later steps come from the creator (CONCEPT.md).
5. //
6. // the header's ident-selector / "Log in with ident" -> <ident>/login?key=&return=<this app>/login/callback
7. // /login/callback?ident_code=<code> the app's server exchanges the code for the identity's short id
8. // (POST <ident>/api/exchange) and signs this app's OWN session in
9. // (session.user = { id = <users @id> }, users.hl)
10. // the header's "Log out" button signs this app's session out via the face trackerLogOut (components/main.hl)
11. // — ident's own session, and the browser's ident cookie, are untouched
12. //
13. // Config (env; no committed secret — the first deploy registers this app in ident and sets these, done by the
14. // architect, as with ident's own .env for SMTP):
15. // TRACKER_PORT (45008), TRACKER_URL (this app's own public address, for the return URL),
16. // IDENT_URL (https://ident.worldapi.org), TRACKER_KEY (pk_…), TRACKER_SECRET (sk_…)
17. import WebFramework from 'hl:web'
18. import { Response } from 'hl:http1'
19. import { randomBytes } from 'hl:crypto'
20. import { env } from 'hl:proc'
21. import { readBytes, readFile, exists } from 'hl:fs'
22. import { now, timestamp, every } from 'hl:time'
24. import Styles from './styles.hl'
25. import { exchangeCode, ensureUser } from './users.hl'
26. import Home from './components/home.hl'
27. import LoginFailed from './components/loginfailed.hl'
28. import Show from './components/show.hl'
29. import Unwatched from './components/unwatched.hl'
30. import Schedule from './components/schedule.hl'
31. import MyShows from './components/myshows.hl'
32. import MyMovies from './components/mymovies.hl'
33. import Movies from './components/movies.hl'
34. import AllShows from './components/allshows.hl'
35. import AllSeries from './components/allseries.hl'
36. import Search from './components/search.hl'
37. import Person from './components/person.hl'
38. import Genre from './components/genre.hl'
39. import Franchises from './components/franchises.hl'
40. import Franchise from './components/franchise.hl'
41. import FranchiseEdit from './components/franchiseedit.hl'
42. import TimelineEdit from './components/timelineedit.hl'
43. import { ensureIndex, searchTitleChanged, searchTitleRenamed } from './search.hl'
44. import { syncEnabled, syncShow, syncShowDelta, persistSync, emptyTotals, addTotals, totalsLine, pauseMsAfter, msPerRequest, adultBackfillQueue, backfillTitle, kindBackfillQueue, kindTitle } from './tmdbsync.hl'
45. import { detailsRepairQueue, completeStep, completeLine, persistDetails, persistCredits, applySynced, creditsQueue, creditsStep } from './details.hl'
46. import { planStart, planStep, planIds, planLine, markSynced, moveCopiedSummaries, isFollowedInRun } from './deltasync.hl'
47. import { showBySlug, titleKind, titlePath, warmTitleCaches, warmEpisodeKeys } from './shows.hl'
48. import { allWatchedEpisodeIds } from './watches.hl'
49. import { allFollowedShowIds } from './follows.hl'
50. import { refreshCatalogShow } from './catalog.hl'
51. import { ensureShortIds, shortIdStep, shortIdPath, shortIdsLeft, shortIdStats, persistShortIds } from './shortids.hl'
52. import { guestPersonId } from './people.hl'
53. import { personById } from './shows.hl'
54. import { dark, darker } from './shared/tokens.hl'
55. import { ensureFranchises, seedPrepare, seedStep, seedLine, seedPersist } from './franchises.hl'
57. static appTitle = "tracker"
59. // ---- THE INSTALLABLE APP (tracker.worldapi.org#10), the same way calendar.worldapi.org does it: hl:web's own web app
60. // manifest (/__hl/manifest.webmanifest, linked from every head with the apple-touch-icon and theme-color) and service
61. // worker (/__hl/sw.js) from these settings — no JavaScript of ours. Icons in icons/ (icon.svg is the source, the PNGs
62. // are rendered from it with rsvg-convert, README "Icons"). The theme colour is the header's background (darker), the
63. // splash background the page's (dark) — both off the tokens, not written out again.
64. appThemeColor = darker.value
65. appBackgroundColor = dark.value
66. appIcons = [
67. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'any' }
68. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'any' }
69. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'maskable' }
70. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'maskable' }
71. ]
72. appTouchIcon = '/icons/apple-touch-icon.png'
73. appFavicon = '/icons/favicon.svg'
74. // OFFLINE: only the shell — the header and `/` (components/main.hl says "You are offline" while the browser has no
75. // network). The data pages are not kept: without a network they get hl:web's "Unavailable offline" page.
76. offline = [ Home ]
78. styles = Styles
80. port = env('TRACKER_PORT') != null ? toNumber(env('TRACKER_PORT')) : 45008
81. watching = env('TRACKER_WATCH') != '0'
82. // read by hl:web's own manifest config (WebFramework.hl `cfg.sessionDir`), not the constructor call below
83. sessionDir = env('TRACKER_SESSIONS') != null ? env('TRACKER_SESSIONS') : null
85. // ---- THE LOGIN BUTTON'S RETURN (ident README "How apps use ident"), copied from calendar.worldapi.org's project.hl
86. // BACK TO THE PAGE: /login.js puts `?next=` into the button's return URL at the click. Only a same-origin PATH
87. // goes (one `/`, URL-safe characters, ≤ 500). Anything else → `/`.
88. nextChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~/?&=%+,;@!$()*:'
89. safePath = (&want) => {
90. if (want == null || hlTypeName(want) != 'String' || want == '' || want.length > 500) { return '/' }
91. if (want.slice(0, 1) != '/' || want.slice(0, 2) == '//' || want.slice(0, 7) == '/login/') { return '/' }
92. let i = 0
93. while (i < want.length) {
94. if (!nextChars.includes(want[i])) { return '/' }
95. i = i + 1
96. }
97. return want
98. }
100. // A FAILED LOGIN is a page (components/loginfailed.hl): the reason is parked in the session, then → /login/failed
101. failed = (&req, why) => {
102. let s = req.session
103. let fresh = s == null
104. if (fresh) { s = server.sessions.mint() }
105. s.data.loginError = why
106. server.sessions.save(s)
107. let res = new Response('login failed: ' + why, { status = 302 headers = { 'Location' = '/login/failed' 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
108. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
109. return res
110. }
112. // the function route gets the cookie's session as req.session (hybriel #11); none yet → minted here
113. loginCallback = (&route, &req) => {
114. if (req.method != 'GET') { return failed(req, 'GET only') }
115. let q = req.query != null ? req.query : {}
116. let code = q.ident_code
117. if (code == null || code == '') { return failed(req, 'ident sent no login code') }
118. let x = exchangeCode(code)
119. if (x.error != null) { return failed(req, x.error) }
120. let u = ensureUser(x.identity)
121. if (u == null) { return failed(req, 'could not store the user') }
122. let s = req.session
123. let fresh = s == null
124. if (fresh) { s = server.sessions.mint() }
125. s.user = { id = u.id }
126. s.data.tag = randomBytes(16)
127. s.data.loginError = null
128. server.sessions.save(s)
129. let res = new Response('logged in', { status = 302 headers = { 'Location' = safePath(q.next) 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
130. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
131. return res
132. }
134. // ---- /posters/<name> (tracker.worldapi.org#4): a show's poster image, served dynamically from
135. // storage/mpackdb/posters/ (the poster's file name is the show's oldId + its migrated extension,
136. // shows.hl posterName) — falls back to a placeholder svg when the real file isn't there yet (the
137. // mongo export never included the actual poster bytes, see the report's `open`).
138. postersDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/posters'
139. placeholderPoster = './assets/poster-placeholder.svg'
140. posterNameChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_'
141. isSafePosterName = (&name) => {
142. if (name == null || hlTypeName(name) != 'String' || name == '' || name.length > 100) { return false }
143. let i = 0
144. while (i < name.length) {
145. if (!posterNameChars.includes(name[i])) { return false }
146. i = i + 1
147. }
148. return true
149. }
150. endsWithStr = (&s, suffix) => { return s.length >= suffix.length && s.slice(s.length - suffix.length) == suffix }
151. mimeOfPoster = (&name) => {
152. if (endsWithStr(name, '.png')) { return 'image/png' }
153. if (endsWithStr(name, '.webp')) { return 'image/webp' }
154. if (endsWithStr(name, '.svg')) { return 'image/svg+xml' }
155. return 'image/jpeg'
156. }
157. posterRoute = (&route, &req) => {
158. let name = route.params != null ? route.params.name : null
159. if (!isSafePosterName(name)) { return new Response('bad poster name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
160. let path = postersDir + '/' + name
161. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
162. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
163. }
165. // ---- /profiles/<name> (tracker.worldapi.org#16): a person's photo, from storage/mpackdb/profiles/ (people.hl syncProfile
166. // downloads it on the first visit of the person page) — the same checks and placeholder as /posters/<name>
167. profilesDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/profiles'
168. profileRoute = (&route, &req) => {
169. let name = route.params != null ? route.params.name : null
170. if (!isSafePosterName(name)) { return new Response('bad photo name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
171. let path = profilesDir + '/' + name
172. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
173. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
174. }
176. // ---- the old addresses of the personal lists (tracker.worldapi.org#8): they all live under /my/ now —
177. // a permanent redirect so old links keep working (architect's decision)
178. movedTo = (&path) => { return new Response('moved to ' + path, { status = 301 headers = { 'Location' = path 'Content-Type' = 'text/plain; charset=utf-8' } }) }
179. unwatchedMoved = (&route, &req) => { return movedTo('/my/unwatched') }
180. scheduleMoved = (&route, &req) => { return movedTo('/my/schedule') }
181. // mission 058: the show page's genre pills linked `/genre/<x>` (no page there) — the genre pages are `/genres/<x>`
182. genreMoved = (&route, &req) => {
183. let g = route.params != null ? route.params.genre : null
184. return movedTo(isSafePosterName(g) ? '/genres/' + g : '/')
185. }
186. // mission 060 (tracker.worldapi.org#27): `/<shortId>` → 301 to the title's or person's page (shortids.hl); upper case is
187. // read as lower case; anything else is a 404
188. shortIdRoute = (&route, &req) => {
189. let sid = route.params != null ? route.params.sid : null
190. let path = sid != null && hlTypeName(sid) == 'String' ? shortIdPath(sid.toLowerCase()) : null
191. if (path == null) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
192. return movedTo(path)
193. }
195. // mission 062: a GUEST STAR's link (components/show.hl castRowsOf) — guests are no person records until someone opens one:
196. // `/person/tmdb/<tmdbId>?show=<show id>` creates the person (people.hl guestPersonId: name + role from that title, a short id) or
197. // finds the one we have, then 302 → `/person/<slug>` (the page fills from TMDB as for any new person). A function route: the
198. // browser loads it itself (hl:web's client router hands every non-component link to the browser)
199. guestRoute = (&route, &req) => {
200. let t = route.params != null ? route.params.tmdb : null
201. let tmdbId = t != null && hlTypeName(t) == 'String' && t.length <= 10 ? toNumber(t) : null
202. if (tmdbId == null || tmdbId < 1) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
203. let q = req.query != null ? req.query : {}
204. let r = guestPersonId(tmdbId, q.show)
205. let p = r.id != null ? personById(r.id) : null
206. if (p == null || p.urlSegment == null) { return new Response('not found: ' + (r.error != null ? r.error : 'no such person'), { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
207. return new Response('see /person/' + p.urlSegment, { status = 302 headers = { 'Location' = '/person/' + p.urlSegment 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
208. }
210. // ---- /shows/<slug> (tracker.worldapi.org#21, mission 068): since the split only the game / reality / talk / news SHOWS live
211. // here; a series answers 301 to /series/<slug>, a movie to /movies/<slug> (old links and bookmarks keep working). hl:web has no
212. // way for a component route to answer a redirect, so this is a FUNCTION route, and for a show (or an unknown slug: the page's own
213. // "Not found") it renders the title page the way hl:web renders a component route (WebFramework.hl `on http.request`, the
214. // `component` branch): the Show component matched at /series/<slug> — same component, same params — with the cookie's session,
215. // a new one (and its cookie) when the browser brought none. In the browser a link to /shows/<slug> is then a full page load
216. // (the client router only navigates component routes itself); every other title page stays a client-side navigation.
217. showsRoute = (&route, &req) => {
218. let slug = route.params != null ? route.params.slug : null
219. let show = showBySlug(slug)
220. if (show != null && titleKind(show) != 'show') { return movedTo(titlePath(show)) }
221. let m = server.router.match('/series/' + slug)
222. let s = req.session
223. let fresh = s == null
224. if (fresh) { s = server.sessions.mint() }
225. let res = server.page(m, s, server.hostOf(req.headers['host']), server.requestHeaders(req.headers))
226. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
227. return res
228. }
230. routes = [
231. { pattern = "/favicon.ico" file = "./icons/favicon.ico" headers = { 'Cache-Control' = 'no-cache' } }
232. // the installable app (tracker.worldapi.org#10): the icons (manifest and service worker are hl:web's own, from appIcons / offline)
233. { pattern = "/icons/icon-192.png" file = "./icons/icon-192.png" headers = { 'Cache-Control' = 'no-cache' } }
234. { pattern = "/icons/icon-512.png" file = "./icons/icon-512.png" headers = { 'Cache-Control' = 'no-cache' } }
235. { pattern = "/icons/apple-touch-icon.png" file = "./icons/apple-touch-icon.png" headers = { 'Cache-Control' = 'no-cache' } }
236. { pattern = "/icons/favicon.svg" file = "./icons/favicon.svg" headers = { 'Cache-Control' = 'no-cache' } }
237. { pattern = "/login/callback" function = loginCallback }
238. { pattern = "/login/failed" component = LoginFailed }
239. { pattern = "/login.js" file = "./login.js" headers = { 'Cache-Control' = 'no-cache' } }
240. { pattern = "/posters/:name" function = posterRoute }
241. // tracker.worldapi.org#13: every movie / every series, 24 per page — the page is a PATH segment (`/movies/page/2`), a
242. // page component cannot read `?page=` (hybriel#11) — and the movies the user follows
243. { pattern = "/movies" component = Movies }
244. { pattern = "/movies/page/:page" component = Movies }
245. // tracker.worldapi.org#21 (mission 068): series and shows split — /series lists the series, /shows the shows; a title's page
246. // is /movies/<slug>, /series/<slug> or /shows/<slug> (one component; /shows/<slug> is showsRoute above)
247. { pattern = "/movies/:slug" component = Show }
248. { pattern = "/series" component = AllSeries }
249. { pattern = "/series/page/:page" component = AllSeries }
250. { pattern = "/series/:slug" component = Show }
251. { pattern = "/shows" component = AllShows }
252. { pattern = "/shows/page/:page" component = AllShows }
253. { pattern = "/shows/:slug" function = showsRoute }
254. { pattern = "/my/movies" component = MyMovies }
255. { pattern = "/my/unwatched" component = Unwatched }
256. { pattern = "/my/schedule" component = Schedule }
257. // #21: /my/series and /my/shows (one component, components/myshows.hl; any other word there is its "Not found")
258. { pattern = "/my/:tv" component = MyShows }
259. { pattern = "/unwatched" function = unwatchedMoved }
260. { pattern = "/schedule" function = scheduleMoved }
261. // tracker.worldapi.org#14: the search — `/search/<text>` (hl:web gives a page no query string, so not `?q=`)
262. { pattern = "/search/:q?" component = Search }
263. // tracker.worldapi.org#16: a person's page (the show page's cast and the search link here) and their photo
264. { pattern = "/person/tmdb/:tmdb" function = guestRoute }
265. { pattern = "/person/:slug" component = Person }
266. { pattern = "/profiles/:name" function = profileRoute }
267. // mission 058 (tracker.worldapi.org#25): a genre's movies and series, like /movies (the page in the path)
268. { pattern = "/genres/:genre" component = Genre }
269. { pattern = "/genres/:genre/page/:page" component = Genre }
270. { pattern = "/genre/:genre" function = genreMoved }
271. // tracker.worldapi.org#19 (mission 066): franchises and timelines, the creator's editor (`/franchises/edit` before the slug)
272. { pattern = "/franchises" component = Franchises }
273. { pattern = "/franchises/edit" component = FranchiseEdit }
274. { pattern = "/franchises/:slug" component = Franchise }
275. { pattern = "/timelines/:tslug" component = Franchise }
276. { pattern = "/timelines/:slug/edit" component = TimelineEdit }
277. { pattern = "/" component = Home }
278. // mission 060 (tracker.worldapi.org#27): a short id → its page (LAST: every named one-segment route above wins)
279. { pattern = "/:sid" function = shortIdRoute }
280. ]
282. // WHO GETS THE PUSH: the login state reaches the tabs of one session (components/main.hl trackerSignedIn/Out).
283. tagOf = (&session) => { return session != null && session.data != null ? session.data.tag : null }
284. audience = {
285. trackerSignedIn = (tag, &session) => { return tag != null && tagOf(session) == tag }
286. trackerSignedOut = (tag, &session) => { return tag != null && tagOf(session) == tag }
287. }
289. // cookies are per host, not per port: an own name keeps this app's session apart from
290. // ident's own (and from any other worldapi app sharing a dev host), see ident README "Design
291. // tokens" / hybriel#10 hybriel#17.
292. sessionCookie = 'trackersid'
293. // tracker.worldapi.org#14: the search index (search.hl) is built once here at boot (~0.6 s on the real data), not by the
294. // first search
295. searchIndexReady = ensureIndex()
296. // mission 060: every short id in use (one map over shows AND persons) and the records still without one, before the first request
297. shortIdsReady = ensureShortIds()
298. // tracker#30 (mission 025): the slim list caches (shows.hl cardOf / episodeRowsOf) of every followed title, before the first
299. // /my/ load — not filled by that load
300. titleCachesStart = now()
301. titleCachesWarm = warmTitleCaches(allFollowedShowIds())
302. episodeKeysWarm = warmEpisodeKeys(allWatchedEpisodeIds())
303. console.log('list caches: ' + titleCachesWarm + ' followed titles, ' + episodeKeysWarm + ' watched episodes in ' + (now() - titleCachesStart) + ' ms')
304. // tracker#18, ONCE (deltasync.hl moveCopiedSummaries): a `summary` that is only the migrated copy of tmdbSummary is MOVED to
305. // `migratedSummary` (mission 070: kept, not shown) — the current TMDB text shows; the creator's own texts stay
306. summariesMoved = moveCopiedSummaries()
307. if (summariesMoved.ran) { console.log('summaries: moved ' + summariesMoved.moved + ' copies of tmdbSummary to migratedSummary, kept ' + summariesMoved.kept + ' own texts, errors ' + summariesMoved.errors + ', ' + summariesMoved.ms + ' ms') }
308. // mission 066: which timelines every title is in (one scan of the timelines), before the first show page
309. franchisesReady = ensureFranchises()
310. server = new WebFramework(routes = routes, styles = styles, port = port, sessionCookie = sessionCookie, watchMode = watching)
312. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, tmdbsync.hl; tracker#18, deltasync.hl) -------------------------
313. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC). hl:web serves ONE
314. // request at a time, so the run never takes the server for itself: a 1-second clock does ONE step per tick and returns — the
315. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
316. // tracker#18: the run first asks TMDB's and TVmaze's CHANGE LISTS since the last run (deltasync.hl — one list page per step)
317. // and then syncs only the titles of ours that changed, followed ones first — every title stays current, not only the followed
318. // ones. FULL WALK (every followed show, as before #18) on the first run, after a gap > 14 days, or when a change list fails.
319. // A title's step: tmdbsync.hl syncShow (1–3 TMDB requests + maybe its poster, TVmaze) + details.hl applySynced (the record's
320. // title, overview, tagline … and cast/crew from the same details answer). After a step that made n requests the next waits
321. // n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze request (≤ 20 / 10 s, tracker.worldapi.org#12).
322. // The finished run stores its day (deltasync.hl markSynced). Log: "tmdb sync: start, …", the change lists ("tmdb sync:
323. // changes since …"), errors, slow steps, "tmdb sync done: mode=… changeRequests=… <totals>".
324. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
325. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
326. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
327. syncQueue = []
328. syncPlanning = false
329. syncMode = ''
330. syncMark = false
331. syncChangeRequests = 0
332. syncAt = 0
333. syncDay = ''
334. syncNextAt = 0
335. syncTotals = emptyTotals()
336. syncFinish = () => {
337. persistDetails()
338. if (syncMark) { markSynced(syncDay) }
339. console.log('tmdb sync done: mode=' + syncMode + ' changeRequests=' + syncChangeRequests + ' ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000))
340. return null
341. }
342. syncTick = () => {
343. let t = now()
344. if (syncQueue.length == 0 && !syncPlanning) {
345. let stamp = timestamp(t)
346. let day = stamp.slice(0, 10)
347. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
348. syncDay = day
349. syncAt = t
350. syncNextAt = 0
351. syncTotals = emptyTotals()
352. syncChangeRequests = 0
353. let p = planStart(day, null)
354. syncMode = p.mode
355. syncMark = true
356. if (p.mode == 'delta') {
357. syncPlanning = true
358. console.log('tmdb sync: start, changes ' + p.why)
359. return null
360. }
361. syncQueue = allFollowedShowIds()
362. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows, full walk (' + p.why + ')')
363. if (syncQueue.length == 0) { syncFinish() }
364. return null
365. }
366. if (t < syncNextAt) { return null }
367. if (syncPlanning) {
368. let ps = planStep()
369. syncChangeRequests = syncChangeRequests + ps.requests + ps.tvmazeRequests
370. syncTotals.requests = syncTotals.requests + ps.requests
371. syncTotals.tvmazeRequests = syncTotals.tvmazeRequests + ps.tvmazeRequests
372. syncNextAt = now() + pauseMsAfter(ps)
373. if (!ps.done) { return null }
374. syncPlanning = false
375. console.log('tmdb sync: ' + planLine())
376. if (ps.failed != '') {
377. // a change list could not be read: the old full walk, and the day is not advanced (the next run asks again)
378. syncMode = 'full'
379. syncMark = false
380. syncQueue = allFollowedShowIds()
381. console.log('tmdb sync: change list failed, full walk of ' + syncQueue.length + ' followed shows instead')
382. } else {
383. syncQueue = planIds()
384. }
385. if (syncQueue.length == 0) { syncFinish() }
386. return null
387. }
388. let id = syncQueue[0]
389. let rest = []
390. let i = 1
391. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
392. syncQueue = rest
393. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze)
394. let r = syncMode == 'delta' && !isFollowedInRun(id) ? syncShowDelta(id) : syncShow(id)
395. // tracker#18 + #28: the record's own fields (title, overview, tagline …) and the cast + crew from the same details answer
396. if (r.details != null) {
397. let c = applySynced(id, r.details)
398. if (c.error != '') { console.log('tmdb sync: ' + c.error) }
399. if (c.renamed) { searchTitleRenamed(id) }
400. }
401. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
402. // also brings TMDB's adult flag (mission 054) → the search's view of it too
403. refreshCatalogShow(id)
404. searchTitleChanged(id)
405. let took = now() - t
406. syncTotals = addTotals(syncTotals, r)
407. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
408. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
409. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
410. syncNextAt = now() + pauseMsAfter(r)
411. if (syncQueue.length > 0 && syncTotals.shows % 10 == 0) { persistDetails() }
412. if (syncQueue.length == 0) { syncFinish() }
413. return null
414. }
416. // ---- THE ADULT-FLAG BACKFILL (mission 054, tmdbsync.hl backfillTitle) -----------------------------------
417. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
418. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
419. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
420. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
421. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
422. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
423. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
424. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
425. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
426. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
427. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
428. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
429. backfillQueue = null
430. backfillPos = 0
431. backfillDone = false
432. backfillAt = 0
433. backfillNextAt = 0
434. backfillTries = {}
435. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
436. backfillLine = () => {
437. let b = backfillTotals
438. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
439. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
440. }
441. backfillTick = () => {
442. if (backfillDone) { return null }
443. let t = now()
444. if (backfillQueue == null) {
445. backfillQueue = adultBackfillQueue()
446. backfillAt = t
447. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
448. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
449. return null
450. }
451. if (t < backfillNextAt) { return null }
452. let id = backfillQueue[backfillPos]
453. backfillPos = backfillPos + 1
454. let r = backfillTitle(id)
455. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
456. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
457. let q = backfillQueue
458. q.push(id)
459. backfillQueue = q
460. backfillTotals.retries = backfillTotals.retries + 1
461. } else {
462. backfillTotals.titles = backfillTotals.titles + 1
463. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
464. }
465. backfillTotals.posters = backfillTotals.posters + r.posters
466. backfillTotals.ids = backfillTotals.ids + r.ids
467. backfillTotals.requests = backfillTotals.requests + r.requests
468. backfillTotals.errors = backfillTotals.errors + r.errors
469. if (r.error != '') { console.log('adult backfill: ' + r.error) }
470. if (r.adult != null) {
471. refreshCatalogShow(id)
472. searchTitleChanged(id)
473. }
474. let took = now() - t
475. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
476. backfillNextAt = now() + r.requests * msPerRequest
477. let finished = backfillPos >= backfillQueue.length
478. if (finished || backfillPos % 25 == 0) { persistSync() }
479. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
480. if (finished) {
481. backfillDone = true
482. console.log('adult backfill done: ' + backfillLine())
483. }
484. return null
485. }
487. // ---- THE DETAILS REPAIR (tracker.worldapi.org#26, details.hl completeStep) ----------------------------------------
488. // Titles the person pages added from a filmography (#16) are MINIMAL (no poster, seasons, ids, cast …). Opening one completes
489. // it (components/show.hl); this job completes the rest in the background: at start it collects every incomplete, non-adult
490. // title and does ONE per step on the app's clock, like the adult backfill — a step is the daily sync's work for one title
491. // (1–3 TMDB requests + poster + TVmaze), requests queued meanwhile are served between two steps; then a pause of 260 ms per
492. // TMDB request + 550 ms per TVmaze request (pauseMsAfter: ≤ 40 / 10 s, ≤ 20 / 10 s). RESUMES after a restart (the queue is
493. // what is still incomplete); TMDB 404 marks the title (`detailsCheck`), a passing failure goes to the back of the queue (at
494. // most 2 more tries per start). Waits while the daily sync or the adult backfill runs. Persisted every 10 titles + at the end.
495. // Log: start, every TRACKER_REPAIR_LOG_EVERY (100) titles, each failure, done. TRACKER_REPAIR=0 switches it off (off too
496. // without TMDB_READ_TOKEN). TRACKER_REPAIR_FAST=1 (the gate, fake TMDB only): 20 ms pause instead.
497. repairOn = env('TRACKER_REPAIR') != '0' && syncEnabled
498. repairFast = env('TRACKER_REPAIR_FAST') == '1'
499. repairLogEvery = env('TRACKER_REPAIR_LOG_EVERY') != null ? toNumber(env('TRACKER_REPAIR_LOG_EVERY')) : 100
500. repairQueue = null
501. repairPos = 0
502. repairDone = false
503. repairAt = 0
504. repairNextAt = 0
505. repairTries = {}
506. repairTotals = { titles = 0 complete = 0 failed = 0 adult = 0 newSeasons = 0 newEpisodes = 0 posters = 0 cast = 0 requests = 0 tvmazeRequests = 0 retries = 0 }
507. repairLine = () => {
508. let b = repairTotals
509. return b.titles + '/' + repairQueue.length + ' titles, complete=' + b.complete + ' failed=' + b.failed + ' adult=' + b.adult + ' newSeasons=' + b.newSeasons + ' newEpisodes=' + b.newEpisodes
510. + ' posters=' + b.posters + ' cast=' + b.cast + ' requests=' + b.requests + ' tvmazeRequests=' + b.tvmazeRequests + ' retries=' + b.retries + ' seconds=' + ((now() - repairAt) / 1000)
511. }
512. repairTick = () => {
513. if (repairDone) { return null }
514. let t = now()
515. if (repairQueue == null) {
516. repairQueue = detailsRepairQueue()
517. repairAt = t
518. console.log('details repair: start, ' + repairQueue.length + ' incomplete titles (' + (now() - t) + ' ms to find them)')
519. if (repairQueue.length == 0) { repairDone = true console.log('details repair: nothing to do') }
520. return null
521. }
522. if (t < repairNextAt) { return null }
523. let id = repairQueue[repairPos]
524. let r = completeStep(id)
525. let took = now() - t
526. if (took > 1500) { console.log('details repair: slow step, show ' + id + ', ' + took + ' ms') }
527. if (r.error != '') { console.log('details repair: ' + r.error) }
528. repairNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
529. // a series takes several steps: the same title again next time
530. if (!r.done && !r.retry) { return null }
531. repairPos = repairPos + 1
532. if (r.retry && (repairTries[id] == null || repairTries[id] < 2)) {
533. repairTries[id] = repairTries[id] == null ? 1 : repairTries[id] + 1
534. let q = repairQueue
535. q.push(id)
536. repairQueue = q
537. repairTotals.retries = repairTotals.retries + 1
538. } else if (!r.skipped) {
539. // done: its counts (a failed first step: the request alone)
540. let sum = r.sum
541. repairTotals.titles = repairTotals.titles + 1
542. if (r.adult == true) { repairTotals.adult = repairTotals.adult + 1 } else if (sum.steps == 0) { repairTotals.failed = repairTotals.failed + 1 } else { repairTotals.complete = repairTotals.complete + 1 }
543. repairTotals.newSeasons = repairTotals.newSeasons + sum.newSeasons
544. repairTotals.newEpisodes = repairTotals.newEpisodes + sum.newEpisodes
545. repairTotals.posters = repairTotals.posters + sum.posters
546. repairTotals.cast = repairTotals.cast + sum.cast
547. repairTotals.requests = repairTotals.requests + (sum.steps == 0 ? r.requests : sum.requests)
548. repairTotals.tvmazeRequests = repairTotals.tvmazeRequests + sum.tvmazeRequests
549. }
550. let finished = repairPos >= repairQueue.length
551. if (finished || repairPos % 10 == 0) { persistDetails() }
552. if (!finished && !r.skipped && !r.retry && repairTotals.titles > 0 && repairTotals.titles % repairLogEvery == 0) { console.log('details repair: ' + repairLine()) }
553. if (finished) {
554. repairDone = true
555. console.log('details repair done: ' + repairLine())
556. }
557. return null
558. }
560. // ---- THE KIND BACKFILL (tracker.worldapi.org#21, mission 068, tmdbsync.hl kindTitle) ----------------------------------
561. // Every TV title without a stored kind gets one: a title with a show genre (Reality / Talk / News) from the stored data (no
562. // request), any other from TMDB's TV type (ONE request: GET /tv/<id>, then 260 ms pause). Starts once the adult backfill is
563. // done (or off), waits while the daily sync runs, ONE title per step on the same clock. A title that becomes a SHOW moves from
564. // /series to /shows at once (catalog.hl refreshCatalogShow). RESUMES after a restart (the queue = every TV title still without
565. // a kind); a 404 stores the genres' answer + `kindCheck`; a passing failure goes to the back of the queue, at most 2 more tries
566. // per start. Log: start, every TRACKER_KINDS_LOG_EVERY (500) titles, done. TRACKER_KINDS=0 switches it off; without
567. // TMDB_READ_TOKEN it is off too (the lists still split by genre: shows.hl titleKind).
568. kindsOn = env('TRACKER_KINDS') != '0' && syncEnabled
569. kindsLogEvery = env('TRACKER_KINDS_LOG_EVERY') != null ? toNumber(env('TRACKER_KINDS_LOG_EVERY')) : 500
570. kindQueue = null
571. kindPos = 0
572. kindDone = false
573. kindAt = 0
574. kindNextAt = 0
575. kindTries = {}
576. kindTotals = { titles = 0 series = 0 shows = 0 fromGenre = 0 moved = 0 requests = 0 errors = 0 retries = 0 }
577. kindLine = () => {
578. let k = kindTotals
579. return k.titles + '/' + kindQueue.length + ' titles, series=' + k.series + ' shows=' + k.shows + ' fromGenre=' + k.fromGenre + ' moved=' + k.moved
580. + ' requests=' + k.requests + ' errors=' + k.errors + ' retries=' + k.retries + ' seconds=' + ((now() - kindAt) / 1000)
581. }
582. kindTick = () => {
583. if (kindDone) { return null }
584. let t = now()
585. if (kindQueue == null) {
586. kindQueue = kindBackfillQueue()
587. kindAt = t
588. console.log('kind backfill: start, ' + kindQueue.length + ' TV titles without a kind (' + (now() - t) + ' ms to find them)')
589. if (kindQueue.length == 0) { kindDone = true console.log('kind backfill: nothing to do') }
590. return null
591. }
592. if (t < kindNextAt) { return null }
593. let id = kindQueue[kindPos]
594. kindPos = kindPos + 1
595. let r = kindTitle(id)
596. if (r.retry && (kindTries[id] == null || kindTries[id] < 2)) {
597. kindTries[id] = kindTries[id] == null ? 1 : kindTries[id] + 1
598. let q = kindQueue
599. q.push(id)
600. kindQueue = q
601. kindTotals.retries = kindTotals.retries + 1
602. } else {
603. kindTotals.titles = kindTotals.titles + 1
604. if (r.kind == 'show') { kindTotals.shows = kindTotals.shows + 1 } else if (r.kind == 'series') { kindTotals.series = kindTotals.series + 1 }
605. if (r.fromGenre) { kindTotals.fromGenre = kindTotals.fromGenre + 1 }
606. }
607. kindTotals.requests = kindTotals.requests + r.requests
608. kindTotals.errors = kindTotals.errors + r.errors
609. if (r.error != '') { console.log('kind backfill: ' + r.error) }
610. // until now the genres decided (no show genre = series): only a title TMDB's type makes a SHOW changes its list
611. if (r.kind == 'show' && !r.fromGenre) {
612. refreshCatalogShow(id)
613. kindTotals.moved = kindTotals.moved + 1
614. }
615. let took = now() - t
616. if (took > 1500) { console.log('kind backfill: slow step, show ' + id + ', ' + took + ' ms') }
617. kindNextAt = now() + r.requests * msPerRequest
618. let finished = kindPos >= kindQueue.length
619. if (finished || kindPos % 25 == 0) { persistSync() }
620. if (!finished && kindTotals.titles > 0 && kindTotals.titles % kindsLogEvery == 0 && !r.retry) { console.log('kind backfill: ' + kindLine()) }
621. if (finished) {
622. kindDone = true
623. console.log('kind backfill done: ' + kindLine())
624. }
625. return null
626. }
628. // ---- THE CREDITS JOB (tracker.worldapi.org#28, details.hl creditsStep) ------------------------------------------------------
629. // Every PUBLIC title (adult == false; adult/unknown are skipped) with a TMDB id gets its whole cast + crew once: at start (after
630. // the adult backfill and the details repair are done) the job collects every such title without `creditsAt` and asks ONE per
631. // step on the app's clock — one request (details + credits / aggregate_credits), then 260 ms (≤ 40 / 10 s); requests queued
632. // meanwhile are served between two steps. Titles the details repair, the daily sync or an import already did carry `creditsAt`
633. // and are not asked. RESUMES after a restart (the queue = what still has no credits); TMDB 404 / adult → `creditsCheck`, never
634. // asked again; a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily sync
635. // runs. Persisted (shows + people only) every 25 titles + at the end. Log: start, every TRACKER_CREDITS_LOG_EVERY (500) titles, failures, slow steps
636. // (> 1.5 s), done. TRACKER_CREDITS=0 switches it off (off too without TMDB_READ_TOKEN). TRACKER_CREDITS_FAST=1 (the gate, fake
637. // TMDB only): 20 ms pause instead.
638. // MEMORY LIMIT: the runtime never frees what a fetch + JSON parse took (mission 061 measured ~6 MB more RSS per title on the real
639. // data, 300 fetches of a 440 KB JSON = +2.7 GB in a 10-line script) — the whole job in one process would add ~35 GB. Before every
640. // step the job reads its own RSS (/proc/self/status); above TRACKER_CREDITS_MAX_MB (16000) it PAUSES for this process (logged) and
641. // goes on with what is left after the next restart (deploy, `docker restart`).
642. creditsOn = env('TRACKER_CREDITS') != '0' && syncEnabled
643. creditsFast = env('TRACKER_CREDITS_FAST') == '1'
644. creditsLogEvery = env('TRACKER_CREDITS_LOG_EVERY') != null ? toNumber(env('TRACKER_CREDITS_LOG_EVERY')) : 500
645. creditsQueueIds = null
646. creditsPos = 0
647. creditsDone = false
648. creditsAt = 0
649. creditsNextAt = 0
650. creditsTries = {}
651. creditsMaxMb = env('TRACKER_CREDITS_MAX_MB') != null ? toNumber(env('TRACKER_CREDITS_MAX_MB')) : 16000
652. creditsTotals = { titles = 0 done = 0 failed = 0 cast = 0 crew = 0 people = 0 requests = 0 retries = 0 slowest = 0 }
653. // this process' resident memory in MB (VmRSS), 0 when unknown
654. rssMb = () => {
655. if (!exists('/proc/self/status')) { return 0 }
656. let t = readFile('/proc/self/status')
657. let i = t != null ? t.indexOf('VmRSS:') : -1
658. if (i < 0) { return 0 }
659. let kb = toNumber(t.slice(i + 6, t.indexOf('kB', i)).trim())
660. return kb != null ? (kb - kb % 1024) / 1024 : 0
661. }
662. creditsLine = () => {
663. let b = creditsTotals
664. return b.titles + '/' + creditsQueueIds.length + ' titles, done=' + b.done + ' failed=' + b.failed + ' cast=' + b.cast + ' crew=' + b.crew + ' people=' + b.people
665. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - creditsAt) / 1000)
666. }
667. creditsTick = () => {
668. if (creditsDone) { return null }
669. let t = now()
670. if (creditsQueueIds == null) {
671. creditsQueueIds = creditsQueue()
672. creditsAt = t
673. console.log('credits job: start, ' + creditsQueueIds.length + ' titles without credits (' + (now() - t) + ' ms to find them)')
674. if (creditsQueueIds.length == 0) { creditsDone = true console.log('credits job: nothing to do') }
675. return null
676. }
677. if (t < creditsNextAt) { return null }
678. let mb = rssMb()
679. if (mb > creditsMaxMb) {
680. creditsDone = true
681. persistCredits()
682. console.log('credits job: paused, RSS ' + mb + ' MB > TRACKER_CREDITS_MAX_MB ' + creditsMaxMb + ' — ' + (creditsQueueIds.length - creditsPos) + ' titles left for the next start; ' + creditsLine())
683. return null
684. }
685. let id = creditsQueueIds[creditsPos]
686. creditsPos = creditsPos + 1
687. let r = creditsStep(id)
688. let took = now() - t
689. if (took > creditsTotals.slowest) { creditsTotals.slowest = took }
690. if (took > 1500) { console.log('credits job: slow step, show ' + id + ', ' + took + ' ms, cast=' + r.cast + ' crew=' + r.crew + ' people=' + r.people) }
691. if (r.error != '') { console.log('credits job: ' + r.error) }
692. creditsNextAt = now() + (creditsFast ? 20 : r.requests * msPerRequest)
693. if (r.retry && (creditsTries[id] == null || creditsTries[id] < 2)) {
694. creditsTries[id] = creditsTries[id] == null ? 1 : creditsTries[id] + 1
695. let q = creditsQueueIds
696. q.push(id)
697. creditsQueueIds = q
698. creditsTotals.retries = creditsTotals.retries + 1
699. } else if (!r.skipped) {
700. creditsTotals.titles = creditsTotals.titles + 1
701. if (r.error != '') { creditsTotals.failed = creditsTotals.failed + 1 } else { creditsTotals.done = creditsTotals.done + 1 }
702. creditsTotals.cast = creditsTotals.cast + r.cast
703. creditsTotals.crew = creditsTotals.crew + r.crew
704. creditsTotals.people = creditsTotals.people + r.people
705. }
706. creditsTotals.requests = creditsTotals.requests + r.requests
707. let finished = creditsPos >= creditsQueueIds.length
708. if (finished || creditsPos % 25 == 0) { persistCredits() }
709. if (!finished && !r.skipped && !r.retry && creditsTotals.titles > 0 && creditsTotals.titles % creditsLogEvery == 0) { console.log('credits job: ' + creditsLine()) }
710. if (finished) {
711. creditsDone = true
712. console.log('credits job done: ' + creditsLine())
713. }
714. return null
715. }
717. // ONE clock for all five (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in
718. // between, only while no daily run is going on; the details repair only when neither has work; then the kind backfill (#21);
719. // the credits job (#28) last
720. clockOn = syncOn || backfillOn || repairOn || kindsOn || creditsOn
721. clockTicks = 0
722. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
723. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
724. if (!repairOn) { console.log('details repair: off (' + (syncEnabled ? 'TRACKER_REPAIR=0' : 'no TMDB_READ_TOKEN') + ')') }
725. if (!creditsOn) { console.log('credits job: off (' + (syncEnabled ? 'TRACKER_CREDITS=0' : 'no TMDB_READ_TOKEN') + ')') }
726. if (!kindsOn) { console.log('kind backfill: off (' + (syncEnabled ? 'TRACKER_KINDS=0' : 'no TMDB_READ_TOKEN') + ')') }
727. if (clockOn) {
728. syncClock = every(0.1)
729. on syncClock.tick(x) {
730. clockTicks = clockTicks + 1
731. let busy = syncQueue.length > 0 || syncPlanning
732. if (syncOn && clockTicks % 10 == 0) { syncTick() }
733. let free = !busy && syncQueue.length == 0 && !syncPlanning
734. if (backfillOn && free && !backfillDone) { backfillTick() }
735. else if (repairOn && free && (!backfillOn || backfillDone) && !repairDone) { repairTick() }
736. else if (kindsOn && free && (!backfillOn || backfillDone) && (!repairOn || repairDone) && !kindDone) { kindTick() }
737. else if (creditsOn && free && (!backfillOn || backfillDone) && (!repairOn || repairDone) && (!kindsOn || kindDone)) { creditsTick() }
738. return null
739. }
740. }
742. // ---- THE SHORT-ID BACKFILL (mission 060, tracker.worldapi.org#27, shortids.hl shortIdStep) ----------------------------
743. // Every show and person without a short id gets one in the background: on its own clock (every 0.1 s) ONE batch per tick,
744. // as many records as fit into TRACKER_SHORTID_BUDGET_MS (default 40 ms) — no TMDB, only the tables — so a page waits at
745. // most one batch. The queue is built at start (`ensureShortIds`) from what is still missing = it resumes after a restart.
746. // Persisted every 20 batches and at the end. Log: start, every TRACKER_SHORTID_LOG_EVERY (5000) records, done; with nothing
747. // to do one line. TRACKER_SHORTIDS=0 switches it off (the gate's resume check).
748. shortIdOn = env('TRACKER_SHORTIDS') != '0'
749. shortIdBudget = env('TRACKER_SHORTID_BUDGET_MS') != null ? toNumber(env('TRACKER_SHORTID_BUDGET_MS')) : 40
750. shortIdLogEvery = env('TRACKER_SHORTID_LOG_EVERY') != null ? toNumber(env('TRACKER_SHORTID_LOG_EVERY')) : 5000
751. shortIdRun = { at = 0 batches = 0 assigned = 0 nextLog = 0 slowest = 0 slowestPersist = 0 done = false }
752. shortIdTick = () => {
753. if (shortIdRun.done) { return null }
754. let t = now()
755. let r = shortIdStep(shortIdBudget)
756. let took = now() - t
757. shortIdRun.batches = shortIdRun.batches + 1
758. shortIdRun.assigned = shortIdRun.assigned + r.assigned
759. if (took > shortIdRun.slowest) { shortIdRun.slowest = took }
760. if (r.done || shortIdRun.batches % 20 == 0) {
761. let tp = now()
762. persistShortIds()
763. if (now() - tp > shortIdRun.slowestPersist) { shortIdRun.slowestPersist = now() - tp }
764. }
765. if (!r.done && shortIdRun.assigned >= shortIdRun.nextLog) {
766. console.log('short ids: ' + shortIdRun.assigned + ' assigned, ' + r.left + ' left, ' + ((now() - shortIdRun.at) / 1000) + ' s')
767. shortIdRun.nextLog = shortIdRun.nextLog + shortIdLogEvery
768. }
769. if (r.done) {
770. shortIdRun.done = true
771. let s = shortIdStats()
772. console.log('short ids done: assigned=' + shortIdRun.assigned + ' oldKept=' + s.oldKept + ' batches=' + shortIdRun.batches + ' slowestBatchMs=' + shortIdRun.slowest + ' slowestPersistMs=' + shortIdRun.slowestPersist + ' seconds=' + ((now() - shortIdRun.at) / 1000))
773. }
774. return null
775. }
776. if (!shortIdOn) { console.log('short ids: backfill off (TRACKER_SHORTIDS=0), ' + shortIdsLeft() + ' records without one') }
777. else if (shortIdsLeft() == 0) { console.log('short ids: nothing to do') }
778. else {
779. shortIdRun.at = now()
780. shortIdRun.nextLog = shortIdLogEvery
781. console.log('short ids: backfill start, ' + shortIdsLeft() + ' records, ' + shortIdBudget + ' ms per batch')
782. shortIdClock = every(0.1)
783. on shortIdClock.tick(x) {
784. shortIdTick()
785. return null
786. }
787. }
789. // ---- THE COLLECTION SEED (mission 066, tracker.worldapi.org#19, franchises.hl seedStep) ------------------------------------
790. // TMDB's movie collections become timelines: ONE movie per step (its details; once per collection the collection's parts), on
791. // its own 0.1 s clock like the adult backfill — a pause of 260 ms per TMDB request made (≤ 40 / 10 s). It WAITS while the daily
792. // sync or the adult backfill runs (they share TMDB's limit and the backfill brings the adult flags it skips by); since the merge
793. // (mission 070) also until the details repair, the kind backfill and the credits job are done (or paused) — the LAST job. The queue =
794. // every non-adult movie without a check row, so it RESUMES after a restart and does nothing once all are checked. Persisted
795. // every TRACKER_COLLECTIONS_PERSIST_EVERY (25) steps and at the end. Log: start, every TRACKER_COLLECTIONS_LOG_EVERY (500) movies, done.
796. // TRACKER_COLLECTIONS=0 switches it off; without TMDB_READ_TOKEN it is off too.
797. collectionsOn = env('TRACKER_COLLECTIONS') != '0' && syncEnabled
798. collectionsLogEvery = env('TRACKER_COLLECTIONS_LOG_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_LOG_EVERY')) : 500
799. collectionsPersistEvery = env('TRACKER_COLLECTIONS_PERSIST_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_PERSIST_EVERY')) : 25
800. collectionRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
801. collectionTick = () => {
802. if (collectionRun.done) { return null }
803. // mission 070 (merge): also behind the details repair, the kind backfill and the credits job — one TMDB pacing at a time
804. if (syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && !repairDone) || (kindsOn && !kindDone) || (creditsOn && !creditsDone)) { return null }
805. let t = now()
806. if (!collectionRun.started) {
807. collectionRun.started = true
808. collectionRun.at = t
809. let n = seedPrepare()
810. console.log('collections: start, ' + n + ' movies to check (' + (now() - t) + ' ms to find them)')
811. if (n == 0) { collectionRun.done = true console.log('collections: nothing to do') }
812. return null
813. }
814. if (t < collectionRun.nextAt) { return null }
815. let r = seedStep()
816. if (r.done) {
817. collectionRun.done = true
818. seedPersist()
819. console.log('collections done: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000) + ' slowestStepMs=' + collectionRun.slowest)
820. return null
821. }
822. collectionRun.steps = collectionRun.steps + 1
823. let took = now() - t
824. if (took > collectionRun.slowest) { collectionRun.slowest = took }
825. if (r.error != '') { console.log('collections: ' + r.error) }
826. if (took > 1500) { console.log('collections: slow step, ' + took + ' ms') }
827. collectionRun.nextAt = now() + r.requests * msPerRequest
828. if (collectionRun.steps % collectionsPersistEvery == 0) { seedPersist() }
829. if (collectionRun.steps % collectionsLogEvery == 0) { console.log('collections: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000)) }
830. return null
831. }
832. if (!collectionsOn) { console.log('collections: off (' + (syncEnabled ? 'TRACKER_COLLECTIONS=0' : 'no TMDB_READ_TOKEN') + ')') }
833. else {
834. collectionClock = every(0.1)
835. on collectionClock.tick(x) {
836. collectionTick()
837. return null
838. }
839. }
841. on Error(e) { console.log('error absorbed: ' + e.message) }