tracker

52.6 KB

1. // project.hl — tracker.worldapi.org: STEP 1 (tracker.worldapi.org#1), an empty shell. Login copied unchanged from
2. // calendar.worldapi.org (rejected once for a centered sign-in and no header selector — architect, 2026-09-27):
3. // ident only, no own passwords (README "How apps use ident" of ident.worldapi.org), the identity selector in the
4. // header (components/main.hl). No shows, no data yet — later steps come from the creator (CONCEPT.md).
5. //
6. // the header's ident-selector / "Log in with ident" -> <ident>/login?key=&return=<this app>/login/callback
7. // /login/callback?ident_code=<code> the app's server exchanges the code for the identity's short id
8. // (POST <ident>/api/exchange) and signs this app's OWN session in
9. // (session.user = { id = <users @id> }, users.hl)
10. // the header's "Log out" button signs this app's session out via the face trackerLogOut (components/main.hl)
11. // — ident's own session, and the browser's ident cookie, are untouched
12. //
13. // Config (env; no committed secret — the first deploy registers this app in ident and sets these, done by the
14. // architect, as with ident's own .env for SMTP):
15. // TRACKER_PORT (45008), TRACKER_URL (this app's own public address, for the return URL),
16. // IDENT_URL (https://ident.worldapi.org), TRACKER_KEY (pk_…), TRACKER_SECRET (sk_…)
17. import WebFramework from 'hl:web'
18. import { Response } from 'hl:http1'
19. import { randomBytes } from 'hl:crypto'
20. import { env } from 'hl:proc'
21. import { readBytes, readFile, exists } from 'hl:fs'
22. import { now, timestamp, every } from 'hl:time'
24. import Styles from './styles.hl'
25. import { exchangeCode, ensureUser } from './users.hl'
26. import Home from './components/home.hl'
27. import LoginFailed from './components/loginfailed.hl'
28. import Show from './components/show.hl'
29. import Unwatched from './components/unwatched.hl'
30. import Schedule from './components/schedule.hl'
31. import MyShows from './components/myshows.hl'
32. import MyMovies from './components/mymovies.hl'
33. import Movies from './components/movies.hl'
34. import AllShows from './components/allshows.hl'
35. import AllSeries from './components/allseries.hl'
36. import Search from './components/search.hl'
37. import Person from './components/person.hl'
38. import Genre from './components/genre.hl'
39. import Franchises from './components/franchises.hl'
40. import Franchise from './components/franchise.hl'
41. import FranchiseEdit from './components/franchiseedit.hl'
42. import TimelineEdit from './components/timelineedit.hl'
43. import { ensureIndex, searchTitleChanged, searchTitleRenamed } from './search.hl'
44. import { syncEnabled, syncShow, syncShowDelta, persistSync, emptyTotals, addTotals, totalsLine, pauseMsAfter, msPerRequest, adultBackfillQueue, backfillTitle, kindBackfillQueue, kindTitle } from './tmdbsync.hl'
45. import { detailsRepairQueue, completeStep, completeLine, persistDetails, persistCredits, applySynced, creditsQueue, creditsStep } from './details.hl'
46. import { planStart, planStep, planIds, planLine, markSynced, moveCopiedSummaries, isFollowedInRun } from './deltasync.hl'
47. import { showBySlug, titleKind, titlePath, warmTitleCaches, warmEpisodeKeys, movedSlugs, movedPathOf } from './shows.hl'
48. import { allWatchedEpisodeIds } from './watches.hl'
49. import { allFollowedShowIds } from './follows.hl'
50. import { refreshCatalogShow } from './catalog.hl'
51. import { ensureShortIds, shortIdStep, shortIdPath, shortIdsLeft, shortIdStats, persistShortIds } from './shortids.hl'
52. import { guestPersonId } from './people.hl'
53. import { personById } from './shows.hl'
54. import { dark, darker } from './shared/tokens.hl'
55. import { ensureFranchises, seedPrepare, seedStep, seedLine, seedPersist } from './franchises.hl'
56. import { mergePrepare, mergeStep, mergeStats, mergeLine, persistMerge } from './merge.hl'
58. static appTitle = "tracker"
60. // ---- THE INSTALLABLE APP (tracker.worldapi.org#10), the same way calendar.worldapi.org does it: hl:web's own web app
61. // manifest (/__hl/manifest.webmanifest, linked from every head with the apple-touch-icon and theme-color) and service
62. // worker (/__hl/sw.js) from these settings — no JavaScript of ours. Icons in icons/ (icon.svg is the source, the PNGs
63. // are rendered from it with rsvg-convert, README "Icons"). The theme colour is the header's background (darker), the
64. // splash background the page's (dark) — both off the tokens, not written out again.
65. appThemeColor = darker.value
66. appBackgroundColor = dark.value
67. appIcons = [
68. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'any' }
69. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'any' }
70. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'maskable' }
71. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'maskable' }
72. ]
73. appTouchIcon = '/icons/apple-touch-icon.png'
74. appFavicon = '/icons/favicon.svg'
75. // OFFLINE: only the shell — the header and `/` (components/main.hl says "You are offline" while the browser has no
76. // network). The data pages are not kept: without a network they get hl:web's "Unavailable offline" page.
77. offline = [ Home ]
79. styles = Styles
81. port = env('TRACKER_PORT') != null ? toNumber(env('TRACKER_PORT')) : 45008
82. watching = env('TRACKER_WATCH') != '0'
83. // read by hl:web's own manifest config (WebFramework.hl `cfg.sessionDir`), not the constructor call below
84. sessionDir = env('TRACKER_SESSIONS') != null ? env('TRACKER_SESSIONS') : null
86. // ---- THE LOGIN BUTTON'S RETURN (ident README "How apps use ident"), copied from calendar.worldapi.org's project.hl
87. // BACK TO THE PAGE: /login.js puts `?next=` into the button's return URL at the click. Only a same-origin PATH
88. // goes (one `/`, URL-safe characters, ≤ 500). Anything else → `/`.
89. nextChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~/?&=%+,;@!$()*:'
90. safePath = (&want) => {
91. if (want == null || hlTypeName(want) != 'String' || want == '' || want.length > 500) { return '/' }
92. if (want.slice(0, 1) != '/' || want.slice(0, 2) == '//' || want.slice(0, 7) == '/login/') { return '/' }
93. let i = 0
94. while (i < want.length) {
95. if (!nextChars.includes(want[i])) { return '/' }
96. i = i + 1
97. }
98. return want
99. }
101. // A FAILED LOGIN is a page (components/loginfailed.hl): the reason is parked in the session, then → /login/failed
102. failed = (&req, why) => {
103. let s = req.session
104. let fresh = s == null
105. if (fresh) { s = server.sessions.mint() }
106. s.data.loginError = why
107. server.sessions.save(s)
108. let res = new Response('login failed: ' + why, { status = 302 headers = { 'Location' = '/login/failed' 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
109. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
110. return res
111. }
113. // the function route gets the cookie's session as req.session (hybriel #11); none yet → minted here
114. loginCallback = (&route, &req) => {
115. if (req.method != 'GET') { return failed(req, 'GET only') }
116. let q = req.query != null ? req.query : {}
117. let code = q.ident_code
118. if (code == null || code == '') { return failed(req, 'ident sent no login code') }
119. let x = exchangeCode(code)
120. if (x.error != null) { return failed(req, x.error) }
121. let u = ensureUser(x.identity)
122. if (u == null) { return failed(req, 'could not store the user') }
123. let s = req.session
124. let fresh = s == null
125. if (fresh) { s = server.sessions.mint() }
126. s.user = { id = u.id }
127. s.data.tag = randomBytes(16)
128. s.data.loginError = null
129. server.sessions.save(s)
130. let res = new Response('logged in', { status = 302 headers = { 'Location' = safePath(q.next) 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
131. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
132. return res
133. }
135. // ---- /posters/<name> (tracker.worldapi.org#4): a show's poster image, served dynamically from
136. // storage/mpackdb/posters/ (the poster's file name is the show's oldId + its migrated extension,
137. // shows.hl posterName) — falls back to a placeholder svg when the real file isn't there yet (the
138. // mongo export never included the actual poster bytes, see the report's `open`).
139. postersDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/posters'
140. placeholderPoster = './assets/poster-placeholder.svg'
141. posterNameChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_'
142. isSafePosterName = (&name) => {
143. if (name == null || hlTypeName(name) != 'String' || name == '' || name.length > 100) { return false }
144. let i = 0
145. while (i < name.length) {
146. if (!posterNameChars.includes(name[i])) { return false }
147. i = i + 1
148. }
149. return true
150. }
151. endsWithStr = (&s, suffix) => { return s.length >= suffix.length && s.slice(s.length - suffix.length) == suffix }
152. mimeOfPoster = (&name) => {
153. if (endsWithStr(name, '.png')) { return 'image/png' }
154. if (endsWithStr(name, '.webp')) { return 'image/webp' }
155. if (endsWithStr(name, '.svg')) { return 'image/svg+xml' }
156. return 'image/jpeg'
157. }
158. posterRoute = (&route, &req) => {
159. let name = route.params != null ? route.params.name : null
160. if (!isSafePosterName(name)) { return new Response('bad poster name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
161. let path = postersDir + '/' + name
162. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
163. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
164. }
166. // ---- /profiles/<name> (tracker.worldapi.org#16): a person's photo, from storage/mpackdb/profiles/ (people.hl syncProfile
167. // downloads it on the first visit of the person page) — the same checks and placeholder as /posters/<name>
168. profilesDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/profiles'
169. profileRoute = (&route, &req) => {
170. let name = route.params != null ? route.params.name : null
171. if (!isSafePosterName(name)) { return new Response('bad photo name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
172. let path = profilesDir + '/' + name
173. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
174. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
175. }
177. // ---- the old addresses of the personal lists (tracker.worldapi.org#8): they all live under /my/ now —
178. // a permanent redirect so old links keep working (architect's decision)
179. movedTo = (&path) => { return new Response('moved to ' + path, { status = 301 headers = { 'Location' = path 'Content-Type' = 'text/plain; charset=utf-8' } }) }
180. unwatchedMoved = (&route, &req) => { return movedTo('/my/unwatched') }
181. scheduleMoved = (&route, &req) => { return movedTo('/my/schedule') }
182. // mission 058: the show page's genre pills linked `/genre/<x>` (no page there) — the genre pages are `/genres/<x>`
183. genreMoved = (&route, &req) => {
184. let g = route.params != null ? route.params.genre : null
185. return movedTo(isSafePosterName(g) ? '/genres/' + g : '/')
186. }
187. // mission 060 (tracker.worldapi.org#27): `/<shortId>` → 301 to the title's or person's page (shortids.hl); upper case is
188. // read as lower case; anything else is a 404
189. shortIdRoute = (&route, &req) => {
190. let sid = route.params != null ? route.params.sid : null
191. let path = sid != null && hlTypeName(sid) == 'String' ? shortIdPath(sid.toLowerCase()) : null
192. if (path == null) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
193. return movedTo(path)
194. }
196. // mission 062: a GUEST STAR's link (components/show.hl castRowsOf) — guests are no person records until someone opens one:
197. // `/person/tmdb/<tmdbId>?show=<show id>` creates the person (people.hl guestPersonId: name + role from that title, a short id) or
198. // finds the one we have, then 302 → `/person/<slug>` (the page fills from TMDB as for any new person). A function route: the
199. // browser loads it itself (hl:web's client router hands every non-component link to the browser)
200. guestRoute = (&route, &req) => {
201. let t = route.params != null ? route.params.tmdb : null
202. let tmdbId = t != null && hlTypeName(t) == 'String' && t.length <= 10 ? toNumber(t) : null
203. if (tmdbId == null || tmdbId < 1) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
204. let q = req.query != null ? req.query : {}
205. let r = guestPersonId(tmdbId, q.show)
206. let p = r.id != null ? personById(r.id) : null
207. if (p == null || p.urlSegment == null) { return new Response('not found: ' + (r.error != null ? r.error : 'no such person'), { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
208. return new Response('see /person/' + p.urlSegment, { status = 302 headers = { 'Location' = '/person/' + p.urlSegment 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
209. }
211. // ---- /shows/<slug> (tracker.worldapi.org#21, mission 068): since the split only the game / reality / talk / news SHOWS live
212. // here; a series answers 301 to /series/<slug>, a movie to /movies/<slug> (old links and bookmarks keep working). hl:web has no
213. // way for a component route to answer a redirect, so this is a FUNCTION route, and for a show (or an unknown slug: the page's own
214. // "Not found") it renders the title page the way hl:web renders a component route (WebFramework.hl `on http.request`, the
215. // `component` branch): the Show component matched at /series/<slug> — same component, same params — with the cookie's session,
216. // a new one (and its cookie) when the browser brought none. In the browser a link to /shows/<slug> is then a full page load
217. // (the client router only navigates component routes itself); every other title page stays a client-side navigation.
218. showsRoute = (&route, &req) => {
219. let slug = route.params != null ? route.params.slug : null
220. let show = showBySlug(slug)
221. // tracker#31: a merged duplicate's slug (or a keeper's former one) → the keeper's page
222. if (show != null && (titleKind(show) != 'show' || show.urlSegment != slug)) { return movedTo(titlePath(show)) }
223. let m = server.router.match('/series/' + slug)
224. let s = req.session
225. let fresh = s == null
226. if (fresh) { s = server.sessions.mint() }
227. let res = server.page(m, s, server.hostOf(req.headers['host']), server.requestHeaders(req.headers))
228. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
229. return res
230. }
232. // tracker#31 (mission 026): the slug of a merged duplicate, or a keeper's former slug → 301 to the keeper's page (shows.hl
233. // movedSlugs, read at boot). A component route cannot redirect, so each such slug gets its own function route under /movies/ and
234. // /series/ (/shows/ is showsRoute). A title merged while this process runs answers the keeper's page under its old slug (shows.hl
235. // showBySlug) until the next start.
236. slugMoved = (&route, &req) => {
237. let to = movedPathOf(req.path)
238. if (to == null) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
239. return movedTo(to)
240. }
242. routes = [
243. { pattern = "/favicon.ico" file = "./icons/favicon.ico" headers = { 'Cache-Control' = 'no-cache' } }
244. // the installable app (tracker.worldapi.org#10): the icons (manifest and service worker are hl:web's own, from appIcons / offline)
245. { pattern = "/icons/icon-192.png" file = "./icons/icon-192.png" headers = { 'Cache-Control' = 'no-cache' } }
246. { pattern = "/icons/icon-512.png" file = "./icons/icon-512.png" headers = { 'Cache-Control' = 'no-cache' } }
247. { pattern = "/icons/apple-touch-icon.png" file = "./icons/apple-touch-icon.png" headers = { 'Cache-Control' = 'no-cache' } }
248. { pattern = "/icons/favicon.svg" file = "./icons/favicon.svg" headers = { 'Cache-Control' = 'no-cache' } }
249. { pattern = "/login/callback" function = loginCallback }
250. { pattern = "/login/failed" component = LoginFailed }
251. { pattern = "/login.js" file = "./login.js" headers = { 'Cache-Control' = 'no-cache' } }
252. { pattern = "/posters/:name" function = posterRoute }
253. // tracker.worldapi.org#13: every movie / every series, 24 per page — the page is a PATH segment (`/movies/page/2`), a
254. // page component cannot read `?page=` (hybriel#11) — and the movies the user follows
255. { pattern = "/movies" component = Movies }
256. { pattern = "/movies/page/:page" component = Movies }
257. // tracker.worldapi.org#21 (mission 068): series and shows split — /series lists the series, /shows the shows; a title's page
258. // is /movies/<slug>, /series/<slug> or /shows/<slug> (one component; /shows/<slug> is showsRoute above)
259. { pattern = "/movies/:slug" component = Show }
260. { pattern = "/series" component = AllSeries }
261. { pattern = "/series/page/:page" component = AllSeries }
262. { pattern = "/series/:slug" component = Show }
263. { pattern = "/shows" component = AllShows }
264. { pattern = "/shows/page/:page" component = AllShows }
265. { pattern = "/shows/:slug" function = showsRoute }
266. { pattern = "/my/movies" component = MyMovies }
267. { pattern = "/my/unwatched" component = Unwatched }
268. { pattern = "/my/schedule" component = Schedule }
269. // #21: /my/series and /my/shows (one component, components/myshows.hl; any other word there is its "Not found")
270. { pattern = "/my/:tv" component = MyShows }
271. { pattern = "/unwatched" function = unwatchedMoved }
272. { pattern = "/schedule" function = scheduleMoved }
273. // tracker.worldapi.org#14: the search — `/search/<text>` (hl:web gives a page no query string, so not `?q=`)
274. { pattern = "/search/:q?" component = Search }
275. // tracker.worldapi.org#16: a person's page (the show page's cast and the search link here) and their photo
276. { pattern = "/person/tmdb/:tmdb" function = guestRoute }
277. { pattern = "/person/:slug" component = Person }
278. { pattern = "/profiles/:name" function = profileRoute }
279. // mission 058 (tracker.worldapi.org#25): a genre's movies and series, like /movies (the page in the path)
280. { pattern = "/genres/:genre" component = Genre }
281. { pattern = "/genres/:genre/page/:page" component = Genre }
282. { pattern = "/genre/:genre" function = genreMoved }
283. // tracker.worldapi.org#19 (mission 066): franchises and timelines, the creator's editor (`/franchises/edit` before the slug)
284. { pattern = "/franchises" component = Franchises }
285. { pattern = "/franchises/edit" component = FranchiseEdit }
286. { pattern = "/franchises/:slug" component = Franchise }
287. { pattern = "/timelines/:tslug" component = Franchise }
288. { pattern = "/timelines/:slug/edit" component = TimelineEdit }
289. { pattern = "/" component = Home }
290. // mission 060 (tracker.worldapi.org#27): a short id → its page (LAST: every named one-segment route above wins)
291. { pattern = "/:sid" function = shortIdRoute }
292. ]
294. // WHO GETS THE PUSH: the login state reaches the tabs of one session (components/main.hl trackerSignedIn/Out).
295. tagOf = (&session) => { return session != null && session.data != null ? session.data.tag : null }
296. audience = {
297. trackerSignedIn = (tag, &session) => { return tag != null && tagOf(session) == tag }
298. trackerSignedOut = (tag, &session) => { return tag != null && tagOf(session) == tag }
299. }
301. // cookies are per host, not per port: an own name keeps this app's session apart from
302. // ident's own (and from any other worldapi app sharing a dev host), see ident README "Design
303. // tokens" / hybriel#10 hybriel#17.
304. sessionCookie = 'trackersid'
305. // tracker.worldapi.org#14: the search index (search.hl) is built once here at boot (~0.6 s on the real data), not by the
306. // first search
307. searchIndexReady = ensureIndex()
308. // mission 060: every short id in use (one map over shows AND persons) and the records still without one, before the first request
309. shortIdsReady = ensureShortIds()
310. // tracker#30 (mission 025): the slim list caches (shows.hl cardOf / episodeRowsOf) of every followed title, before the first
311. // /my/ load — not filled by that load
312. titleCachesStart = now()
313. titleCachesWarm = warmTitleCaches(allFollowedShowIds())
314. episodeKeysWarm = warmEpisodeKeys(allWatchedEpisodeIds())
315. console.log('list caches: ' + titleCachesWarm + ' followed titles, ' + episodeKeysWarm + ' watched episodes in ' + (now() - titleCachesStart) + ' ms')
316. // tracker#18, ONCE (deltasync.hl moveCopiedSummaries): a `summary` that is only the migrated copy of tmdbSummary is MOVED to
317. // `migratedSummary` (mission 070: kept, not shown) — the current TMDB text shows; the creator's own texts stay
318. summariesMoved = moveCopiedSummaries()
319. if (summariesMoved.ran) { console.log('summaries: moved ' + summariesMoved.moved + ' copies of tmdbSummary to migratedSummary, kept ' + summariesMoved.kept + ' own texts, errors ' + summariesMoved.errors + ', ' + summariesMoved.ms + ' ms') }
320. // mission 066: which timelines every title is in (one scan of the timelines), before the first show page
321. franchisesReady = ensureFranchises()
322. // tracker#31: the moved slugs' routes first, then `routes` as written (hl:web reads the written list for the pages it serves)
323. movedSlugList = movedSlugs()
324. withMovedSlugs = () => {
325. let out = []
326. for (m of movedSlugList) {
327. out.push({ pattern = '/movies/' + m.slug function = slugMoved })
328. out.push({ pattern = '/series/' + m.slug function = slugMoved })
329. }
330. for (r of routes) { out.push(r) }
331. return out
332. }
333. allRoutes = withMovedSlugs()
334. console.log('moved slugs: ' + movedSlugList.length + ' (301 to the keeper)')
335. server = new WebFramework(routes = allRoutes, styles = styles, port = port, sessionCookie = sessionCookie, watchMode = watching)
337. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, tmdbsync.hl; tracker#18, deltasync.hl) -------------------------
338. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC). hl:web serves ONE
339. // request at a time, so the run never takes the server for itself: a 1-second clock does ONE step per tick and returns — the
340. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
341. // tracker#18: the run first asks TMDB's and TVmaze's CHANGE LISTS since the last run (deltasync.hl — one list page per step)
342. // and then syncs only the titles of ours that changed, followed ones first — every title stays current, not only the followed
343. // ones. FULL WALK (every followed show, as before #18) on the first run, after a gap > 14 days, or when a change list fails.
344. // A title's step: tmdbsync.hl syncShow (1–3 TMDB requests + maybe its poster, TVmaze) + details.hl applySynced (the record's
345. // title, overview, tagline … and cast/crew from the same details answer). After a step that made n requests the next waits
346. // n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze request (≤ 20 / 10 s, tracker.worldapi.org#12).
347. // The finished run stores its day (deltasync.hl markSynced). Log: "tmdb sync: start, …", the change lists ("tmdb sync:
348. // changes since …"), errors, slow steps, "tmdb sync done: mode=… changeRequests=… <totals>".
349. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
350. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
351. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
352. syncQueue = []
353. syncPlanning = false
354. syncMode = ''
355. syncMark = false
356. syncChangeRequests = 0
357. syncAt = 0
358. syncDay = ''
359. syncNextAt = 0
360. syncTotals = emptyTotals()
361. syncFinish = () => {
362. persistDetails()
363. if (syncMark) { markSynced(syncDay) }
364. console.log('tmdb sync done: mode=' + syncMode + ' changeRequests=' + syncChangeRequests + ' ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000))
365. return null
366. }
367. syncTick = () => {
368. let t = now()
369. if (syncQueue.length == 0 && !syncPlanning) {
370. let stamp = timestamp(t)
371. let day = stamp.slice(0, 10)
372. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
373. syncDay = day
374. syncAt = t
375. syncNextAt = 0
376. syncTotals = emptyTotals()
377. syncChangeRequests = 0
378. let p = planStart(day, null)
379. syncMode = p.mode
380. syncMark = true
381. if (p.mode == 'delta') {
382. syncPlanning = true
383. console.log('tmdb sync: start, changes ' + p.why)
384. return null
385. }
386. syncQueue = allFollowedShowIds()
387. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows, full walk (' + p.why + ')')
388. if (syncQueue.length == 0) { syncFinish() }
389. return null
390. }
391. if (t < syncNextAt) { return null }
392. if (syncPlanning) {
393. let ps = planStep()
394. syncChangeRequests = syncChangeRequests + ps.requests + ps.tvmazeRequests
395. syncTotals.requests = syncTotals.requests + ps.requests
396. syncTotals.tvmazeRequests = syncTotals.tvmazeRequests + ps.tvmazeRequests
397. syncNextAt = now() + pauseMsAfter(ps)
398. if (!ps.done) { return null }
399. syncPlanning = false
400. console.log('tmdb sync: ' + planLine())
401. if (ps.failed != '') {
402. // a change list could not be read: the old full walk, and the day is not advanced (the next run asks again)
403. syncMode = 'full'
404. syncMark = false
405. syncQueue = allFollowedShowIds()
406. console.log('tmdb sync: change list failed, full walk of ' + syncQueue.length + ' followed shows instead')
407. } else {
408. syncQueue = planIds()
409. }
410. if (syncQueue.length == 0) { syncFinish() }
411. return null
412. }
413. let id = syncQueue[0]
414. let rest = []
415. let i = 1
416. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
417. syncQueue = rest
418. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze)
419. let r = syncMode == 'delta' && !isFollowedInRun(id) ? syncShowDelta(id) : syncShow(id)
420. // tracker#18 + #28: the record's own fields (title, overview, tagline …) and the cast + crew from the same details answer
421. if (r.details != null) {
422. let c = applySynced(id, r.details)
423. if (c.error != '') { console.log('tmdb sync: ' + c.error) }
424. if (c.renamed) { searchTitleRenamed(id) }
425. }
426. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
427. // also brings TMDB's adult flag (mission 054) → the search's view of it too
428. refreshCatalogShow(id)
429. searchTitleChanged(id)
430. let took = now() - t
431. syncTotals = addTotals(syncTotals, r)
432. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
433. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
434. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
435. syncNextAt = now() + pauseMsAfter(r)
436. if (syncQueue.length > 0 && syncTotals.shows % 10 == 0) { persistDetails() }
437. if (syncQueue.length == 0) { syncFinish() }
438. return null
439. }
441. // ---- THE ADULT-FLAG BACKFILL (mission 054, tmdbsync.hl backfillTitle) -----------------------------------
442. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
443. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
444. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
445. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
446. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
447. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
448. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
449. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
450. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
451. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
452. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
453. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
454. backfillQueue = null
455. backfillPos = 0
456. backfillDone = false
457. backfillAt = 0
458. backfillNextAt = 0
459. backfillTries = {}
460. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
461. backfillLine = () => {
462. let b = backfillTotals
463. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
464. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
465. }
466. backfillTick = () => {
467. if (backfillDone) { return null }
468. let t = now()
469. if (backfillQueue == null) {
470. backfillQueue = adultBackfillQueue()
471. backfillAt = t
472. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
473. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
474. return null
475. }
476. if (t < backfillNextAt) { return null }
477. let id = backfillQueue[backfillPos]
478. backfillPos = backfillPos + 1
479. let r = backfillTitle(id)
480. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
481. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
482. let q = backfillQueue
483. q.push(id)
484. backfillQueue = q
485. backfillTotals.retries = backfillTotals.retries + 1
486. } else {
487. backfillTotals.titles = backfillTotals.titles + 1
488. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
489. }
490. backfillTotals.posters = backfillTotals.posters + r.posters
491. backfillTotals.ids = backfillTotals.ids + r.ids
492. backfillTotals.requests = backfillTotals.requests + r.requests
493. backfillTotals.errors = backfillTotals.errors + r.errors
494. if (r.error != '') { console.log('adult backfill: ' + r.error) }
495. if (r.adult != null) {
496. refreshCatalogShow(id)
497. searchTitleChanged(id)
498. }
499. let took = now() - t
500. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
501. backfillNextAt = now() + r.requests * msPerRequest
502. let finished = backfillPos >= backfillQueue.length
503. if (finished || backfillPos % 25 == 0) { persistSync() }
504. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
505. if (finished) {
506. backfillDone = true
507. console.log('adult backfill done: ' + backfillLine())
508. }
509. return null
510. }
512. // ---- THE DETAILS REPAIR (tracker.worldapi.org#26, details.hl completeStep) ----------------------------------------
513. // Titles the person pages added from a filmography (#16) are MINIMAL (no poster, seasons, ids, cast …). Opening one completes
514. // it (components/show.hl); this job completes the rest in the background: at start it collects every incomplete, non-adult
515. // title and does ONE per step on the app's clock, like the adult backfill — a step is the daily sync's work for one title
516. // (1–3 TMDB requests + poster + TVmaze), requests queued meanwhile are served between two steps; then a pause of 260 ms per
517. // TMDB request + 550 ms per TVmaze request (pauseMsAfter: ≤ 40 / 10 s, ≤ 20 / 10 s). RESUMES after a restart (the queue is
518. // what is still incomplete); TMDB 404 marks the title (`detailsCheck`), a passing failure goes to the back of the queue (at
519. // most 2 more tries per start). Waits while the daily sync or the adult backfill runs. Persisted every 10 titles + at the end.
520. // Log: start, every TRACKER_REPAIR_LOG_EVERY (100) titles, each failure, done. TRACKER_REPAIR=0 switches it off (off too
521. // without TMDB_READ_TOKEN). TRACKER_REPAIR_FAST=1 (the gate, fake TMDB only): 20 ms pause instead.
522. repairOn = env('TRACKER_REPAIR') != '0' && syncEnabled
523. repairFast = env('TRACKER_REPAIR_FAST') == '1'
524. repairLogEvery = env('TRACKER_REPAIR_LOG_EVERY') != null ? toNumber(env('TRACKER_REPAIR_LOG_EVERY')) : 100
525. repairQueue = null
526. repairPos = 0
527. repairDone = false
528. repairAt = 0
529. repairNextAt = 0
530. repairTries = {}
531. repairTotals = { titles = 0 complete = 0 failed = 0 adult = 0 newSeasons = 0 newEpisodes = 0 posters = 0 cast = 0 requests = 0 tvmazeRequests = 0 retries = 0 }
532. repairLine = () => {
533. let b = repairTotals
534. return b.titles + '/' + repairQueue.length + ' titles, complete=' + b.complete + ' failed=' + b.failed + ' adult=' + b.adult + ' newSeasons=' + b.newSeasons + ' newEpisodes=' + b.newEpisodes
535. + ' posters=' + b.posters + ' cast=' + b.cast + ' requests=' + b.requests + ' tvmazeRequests=' + b.tvmazeRequests + ' retries=' + b.retries + ' seconds=' + ((now() - repairAt) / 1000)
536. }
537. repairTick = () => {
538. if (repairDone) { return null }
539. let t = now()
540. if (repairQueue == null) {
541. repairQueue = detailsRepairQueue()
542. repairAt = t
543. console.log('details repair: start, ' + repairQueue.length + ' incomplete titles (' + (now() - t) + ' ms to find them)')
544. if (repairQueue.length == 0) { repairDone = true console.log('details repair: nothing to do') }
545. return null
546. }
547. if (t < repairNextAt) { return null }
548. let id = repairQueue[repairPos]
549. let r = completeStep(id)
550. let took = now() - t
551. if (took > 1500) { console.log('details repair: slow step, show ' + id + ', ' + took + ' ms') }
552. if (r.error != '') { console.log('details repair: ' + r.error) }
553. repairNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
554. // a series takes several steps: the same title again next time
555. if (!r.done && !r.retry) { return null }
556. repairPos = repairPos + 1
557. if (r.retry && (repairTries[id] == null || repairTries[id] < 2)) {
558. repairTries[id] = repairTries[id] == null ? 1 : repairTries[id] + 1
559. let q = repairQueue
560. q.push(id)
561. repairQueue = q
562. repairTotals.retries = repairTotals.retries + 1
563. } else if (!r.skipped) {
564. // done: its counts (a failed first step: the request alone)
565. let sum = r.sum
566. repairTotals.titles = repairTotals.titles + 1
567. if (r.adult == true) { repairTotals.adult = repairTotals.adult + 1 } else if (sum.steps == 0) { repairTotals.failed = repairTotals.failed + 1 } else { repairTotals.complete = repairTotals.complete + 1 }
568. repairTotals.newSeasons = repairTotals.newSeasons + sum.newSeasons
569. repairTotals.newEpisodes = repairTotals.newEpisodes + sum.newEpisodes
570. repairTotals.posters = repairTotals.posters + sum.posters
571. repairTotals.cast = repairTotals.cast + sum.cast
572. repairTotals.requests = repairTotals.requests + (sum.steps == 0 ? r.requests : sum.requests)
573. repairTotals.tvmazeRequests = repairTotals.tvmazeRequests + sum.tvmazeRequests
574. }
575. let finished = repairPos >= repairQueue.length
576. if (finished || repairPos % 10 == 0) { persistDetails() }
577. if (!finished && !r.skipped && !r.retry && repairTotals.titles > 0 && repairTotals.titles % repairLogEvery == 0) { console.log('details repair: ' + repairLine()) }
578. if (finished) {
579. repairDone = true
580. console.log('details repair done: ' + repairLine())
581. }
582. return null
583. }
585. // ---- THE KIND BACKFILL (tracker.worldapi.org#21, mission 068, tmdbsync.hl kindTitle) ----------------------------------
586. // Every TV title without a stored kind gets one: a title with a show genre (Reality / Talk / News) from the stored data (no
587. // request), any other from TMDB's TV type (ONE request: GET /tv/<id>, then 260 ms pause). Starts once the adult backfill is
588. // done (or off), waits while the daily sync runs, ONE title per step on the same clock. A title that becomes a SHOW moves from
589. // /series to /shows at once (catalog.hl refreshCatalogShow). RESUMES after a restart (the queue = every TV title still without
590. // a kind); a 404 stores the genres' answer + `kindCheck`; a passing failure goes to the back of the queue, at most 2 more tries
591. // per start. Log: start, every TRACKER_KINDS_LOG_EVERY (500) titles, done. TRACKER_KINDS=0 switches it off; without
592. // TMDB_READ_TOKEN it is off too (the lists still split by genre: shows.hl titleKind).
593. kindsOn = env('TRACKER_KINDS') != '0' && syncEnabled
594. kindsLogEvery = env('TRACKER_KINDS_LOG_EVERY') != null ? toNumber(env('TRACKER_KINDS_LOG_EVERY')) : 500
595. kindQueue = null
596. kindPos = 0
597. kindDone = false
598. kindAt = 0
599. kindNextAt = 0
600. kindTries = {}
601. kindTotals = { titles = 0 series = 0 shows = 0 fromGenre = 0 moved = 0 requests = 0 errors = 0 retries = 0 }
602. kindLine = () => {
603. let k = kindTotals
604. return k.titles + '/' + kindQueue.length + ' titles, series=' + k.series + ' shows=' + k.shows + ' fromGenre=' + k.fromGenre + ' moved=' + k.moved
605. + ' requests=' + k.requests + ' errors=' + k.errors + ' retries=' + k.retries + ' seconds=' + ((now() - kindAt) / 1000)
606. }
607. kindTick = () => {
608. if (kindDone) { return null }
609. let t = now()
610. if (kindQueue == null) {
611. kindQueue = kindBackfillQueue()
612. kindAt = t
613. console.log('kind backfill: start, ' + kindQueue.length + ' TV titles without a kind (' + (now() - t) + ' ms to find them)')
614. if (kindQueue.length == 0) { kindDone = true console.log('kind backfill: nothing to do') }
615. return null
616. }
617. if (t < kindNextAt) { return null }
618. let id = kindQueue[kindPos]
619. kindPos = kindPos + 1
620. let r = kindTitle(id)
621. if (r.retry && (kindTries[id] == null || kindTries[id] < 2)) {
622. kindTries[id] = kindTries[id] == null ? 1 : kindTries[id] + 1
623. let q = kindQueue
624. q.push(id)
625. kindQueue = q
626. kindTotals.retries = kindTotals.retries + 1
627. } else {
628. kindTotals.titles = kindTotals.titles + 1
629. if (r.kind == 'show') { kindTotals.shows = kindTotals.shows + 1 } else if (r.kind == 'series') { kindTotals.series = kindTotals.series + 1 }
630. if (r.fromGenre) { kindTotals.fromGenre = kindTotals.fromGenre + 1 }
631. }
632. kindTotals.requests = kindTotals.requests + r.requests
633. kindTotals.errors = kindTotals.errors + r.errors
634. if (r.error != '') { console.log('kind backfill: ' + r.error) }
635. // until now the genres decided (no show genre = series): only a title TMDB's type makes a SHOW changes its list
636. if (r.kind == 'show' && !r.fromGenre) {
637. refreshCatalogShow(id)
638. kindTotals.moved = kindTotals.moved + 1
639. }
640. let took = now() - t
641. if (took > 1500) { console.log('kind backfill: slow step, show ' + id + ', ' + took + ' ms') }
642. kindNextAt = now() + r.requests * msPerRequest
643. let finished = kindPos >= kindQueue.length
644. if (finished || kindPos % 25 == 0) { persistSync() }
645. if (!finished && kindTotals.titles > 0 && kindTotals.titles % kindsLogEvery == 0 && !r.retry) { console.log('kind backfill: ' + kindLine()) }
646. if (finished) {
647. kindDone = true
648. console.log('kind backfill done: ' + kindLine())
649. }
650. return null
651. }
653. // ---- THE CREDITS JOB (tracker.worldapi.org#28, details.hl creditsStep) ------------------------------------------------------
654. // Every PUBLIC title (adult == false; adult/unknown are skipped) with a TMDB id gets its whole cast + crew once: at start (after
655. // the adult backfill and the details repair are done) the job collects every such title without `creditsAt` and asks ONE per
656. // step on the app's clock — one request (details + credits / aggregate_credits), then 260 ms (≤ 40 / 10 s); requests queued
657. // meanwhile are served between two steps. Titles the details repair, the daily sync or an import already did carry `creditsAt`
658. // and are not asked. RESUMES after a restart (the queue = what still has no credits); TMDB 404 / adult → `creditsCheck`, never
659. // asked again; a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily sync
660. // runs. Persisted (shows + people only) every 25 titles + at the end. Log: start, every TRACKER_CREDITS_LOG_EVERY (500) titles, failures, slow steps
661. // (> 1.5 s), done. TRACKER_CREDITS=0 switches it off (off too without TMDB_READ_TOKEN). TRACKER_CREDITS_FAST=1 (the gate, fake
662. // TMDB only): 20 ms pause instead.
663. // MEMORY LIMIT: the runtime never frees what a fetch + JSON parse took (mission 061 measured ~6 MB more RSS per title on the real
664. // data, 300 fetches of a 440 KB JSON = +2.7 GB in a 10-line script) — the whole job in one process would add ~35 GB. Before every
665. // step the job reads its own RSS (/proc/self/status); above TRACKER_CREDITS_MAX_MB (16000) it PAUSES for this process (logged) and
666. // goes on with what is left after the next restart (deploy, `docker restart`).
667. creditsOn = env('TRACKER_CREDITS') != '0' && syncEnabled
668. creditsFast = env('TRACKER_CREDITS_FAST') == '1'
669. creditsLogEvery = env('TRACKER_CREDITS_LOG_EVERY') != null ? toNumber(env('TRACKER_CREDITS_LOG_EVERY')) : 500
670. creditsQueueIds = null
671. creditsPos = 0
672. creditsDone = false
673. creditsAt = 0
674. creditsNextAt = 0
675. creditsTries = {}
676. creditsMaxMb = env('TRACKER_CREDITS_MAX_MB') != null ? toNumber(env('TRACKER_CREDITS_MAX_MB')) : 16000
677. creditsTotals = { titles = 0 done = 0 failed = 0 cast = 0 crew = 0 people = 0 requests = 0 retries = 0 slowest = 0 }
678. // this process' resident memory in MB (VmRSS), 0 when unknown
679. rssMb = () => {
680. if (!exists('/proc/self/status')) { return 0 }
681. let t = readFile('/proc/self/status')
682. let i = t != null ? t.indexOf('VmRSS:') : -1
683. if (i < 0) { return 0 }
684. let kb = toNumber(t.slice(i + 6, t.indexOf('kB', i)).trim())
685. return kb != null ? (kb - kb % 1024) / 1024 : 0
686. }
687. creditsLine = () => {
688. let b = creditsTotals
689. return b.titles + '/' + creditsQueueIds.length + ' titles, done=' + b.done + ' failed=' + b.failed + ' cast=' + b.cast + ' crew=' + b.crew + ' people=' + b.people
690. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - creditsAt) / 1000)
691. }
692. creditsTick = () => {
693. if (creditsDone) { return null }
694. let t = now()
695. if (creditsQueueIds == null) {
696. creditsQueueIds = creditsQueue()
697. creditsAt = t
698. console.log('credits job: start, ' + creditsQueueIds.length + ' titles without credits (' + (now() - t) + ' ms to find them)')
699. if (creditsQueueIds.length == 0) { creditsDone = true console.log('credits job: nothing to do') }
700. return null
701. }
702. if (t < creditsNextAt) { return null }
703. let mb = rssMb()
704. if (mb > creditsMaxMb) {
705. creditsDone = true
706. persistCredits()
707. console.log('credits job: paused, RSS ' + mb + ' MB > TRACKER_CREDITS_MAX_MB ' + creditsMaxMb + ' — ' + (creditsQueueIds.length - creditsPos) + ' titles left for the next start; ' + creditsLine())
708. return null
709. }
710. let id = creditsQueueIds[creditsPos]
711. creditsPos = creditsPos + 1
712. let r = creditsStep(id)
713. let took = now() - t
714. if (took > creditsTotals.slowest) { creditsTotals.slowest = took }
715. if (took > 1500) { console.log('credits job: slow step, show ' + id + ', ' + took + ' ms, cast=' + r.cast + ' crew=' + r.crew + ' people=' + r.people) }
716. if (r.error != '') { console.log('credits job: ' + r.error) }
717. creditsNextAt = now() + (creditsFast ? 20 : r.requests * msPerRequest)
718. if (r.retry && (creditsTries[id] == null || creditsTries[id] < 2)) {
719. creditsTries[id] = creditsTries[id] == null ? 1 : creditsTries[id] + 1
720. let q = creditsQueueIds
721. q.push(id)
722. creditsQueueIds = q
723. creditsTotals.retries = creditsTotals.retries + 1
724. } else if (!r.skipped) {
725. creditsTotals.titles = creditsTotals.titles + 1
726. if (r.error != '') { creditsTotals.failed = creditsTotals.failed + 1 } else { creditsTotals.done = creditsTotals.done + 1 }
727. creditsTotals.cast = creditsTotals.cast + r.cast
728. creditsTotals.crew = creditsTotals.crew + r.crew
729. creditsTotals.people = creditsTotals.people + r.people
730. }
731. creditsTotals.requests = creditsTotals.requests + r.requests
732. let finished = creditsPos >= creditsQueueIds.length
733. if (finished || creditsPos % 25 == 0) { persistCredits() }
734. if (!finished && !r.skipped && !r.retry && creditsTotals.titles > 0 && creditsTotals.titles % creditsLogEvery == 0) { console.log('credits job: ' + creditsLine()) }
735. if (finished) {
736. creditsDone = true
737. console.log('credits job done: ' + creditsLine())
738. }
739. return null
740. }
742. // ---- THE DUPLICATE MERGE (tracker.worldapi.org#31, mission 026, merge.hl) ----------------------------------------------------
743. // The old tracker's duplicate titles (same type + TMDB id) become ONE record each — follows, watches, seasons, cast, credits and
744. // timelines moved to the keeper, the others kept as tombstones whose slug and short id lead to it — and seasons that name a title
745. // without being in its list are merged into their twin or linked (merge.hl has the rules). On its own clock (every 0.1 s) ONE
746. // item per tick (a duplicate group, or one title's stray seasons; no TMDB), so pages are served between two items. The TMDB jobs
747. // below (daily sync, backfills, repair, credits, collections) wait until it is done: none of them works on a record that is about
748. // to become a tombstone. RESUMES after a restart (the queue = what is still duplicated / stray). Persisted every 10 items and at
749. // the end. Log: start, one line per tombstone / repaired title, done; with nothing to do one line. TRACKER_MERGE=0 switches it off.
750. mergeOn = env('TRACKER_MERGE') != '0'
751. mergeRun = { started = false done = !mergeOn at = 0 items = 0 slowest = 0 }
752. mergeTick = () => {
753. if (mergeRun.done) { return null }
754. let t = now()
755. if (!mergeRun.started) {
756. mergeRun.started = true
757. mergeRun.at = t
758. let n = mergePrepare()
759. let st = mergeStats()
760. console.log('merge: start, ' + st.groups + ' duplicate groups (' + st.records + ' records), ' + st.strayTitles + ' titles with stray seasons (' + st.ms + ' ms to find them)')
761. if (n == 0) {
762. mergeRun.done = true
763. console.log('merge: nothing to do')
764. }
765. return null
766. }
767. let r = mergeStep()
768. mergeRun.items = mergeRun.items + 1
769. let took = now() - t
770. if (took > mergeRun.slowest) { mergeRun.slowest = took }
771. if (took > 1500) { console.log('merge: slow step, ' + took + ' ms') }
772. if (r.done || mergeRun.items % 10 == 0) { persistMerge() }
773. if (r.done) {
774. mergeRun.done = true
775. console.log('merge done: ' + mergeLine() + ' items=' + mergeRun.items + ' slowestStepMs=' + mergeRun.slowest + ' seconds=' + ((now() - mergeRun.at) / 1000))
776. }
777. return null
778. }
779. if (!mergeOn) { console.log('merge: off (TRACKER_MERGE=0)') }
780. else {
781. mergeClock = every(0.1)
782. on mergeClock.tick(x) {
783. mergeTick()
784. return null
785. }
786. }
788. // ONE clock for all five (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in
789. // between, only while no daily run is going on; the details repair only when neither has work; then the kind backfill (#21);
790. // the credits job (#28) last
791. clockOn = syncOn || backfillOn || repairOn || kindsOn || creditsOn
792. clockTicks = 0
793. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
794. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
795. if (!repairOn) { console.log('details repair: off (' + (syncEnabled ? 'TRACKER_REPAIR=0' : 'no TMDB_READ_TOKEN') + ')') }
796. if (!creditsOn) { console.log('credits job: off (' + (syncEnabled ? 'TRACKER_CREDITS=0' : 'no TMDB_READ_TOKEN') + ')') }
797. if (!kindsOn) { console.log('kind backfill: off (' + (syncEnabled ? 'TRACKER_KINDS=0' : 'no TMDB_READ_TOKEN') + ')') }
798. if (clockOn) {
799. syncClock = every(0.1)
800. on syncClock.tick(x) {
801. clockTicks = clockTicks + 1
802. // tracker#31: not before the duplicate merge is done
803. if (!mergeRun.done) { return null }
804. let busy = syncQueue.length > 0 || syncPlanning
805. if (syncOn && clockTicks % 10 == 0) { syncTick() }
806. let free = !busy && syncQueue.length == 0 && !syncPlanning
807. if (backfillOn && free && !backfillDone) { backfillTick() }
808. else if (repairOn && free && (!backfillOn || backfillDone) && !repairDone) { repairTick() }
809. else if (kindsOn && free && (!backfillOn || backfillDone) && (!repairOn || repairDone) && !kindDone) { kindTick() }
810. else if (creditsOn && free && (!backfillOn || backfillDone) && (!repairOn || repairDone) && (!kindsOn || kindDone)) { creditsTick() }
811. return null
812. }
813. }
815. // ---- THE SHORT-ID BACKFILL (mission 060, tracker.worldapi.org#27, shortids.hl shortIdStep) ----------------------------
816. // Every show and person without a short id gets one in the background: on its own clock (every 0.1 s) ONE batch per tick,
817. // as many records as fit into TRACKER_SHORTID_BUDGET_MS (default 40 ms) — no TMDB, only the tables — so a page waits at
818. // most one batch. The queue is built at start (`ensureShortIds`) from what is still missing = it resumes after a restart.
819. // Persisted every 20 batches and at the end. Log: start, every TRACKER_SHORTID_LOG_EVERY (5000) records, done; with nothing
820. // to do one line. TRACKER_SHORTIDS=0 switches it off (the gate's resume check).
821. shortIdOn = env('TRACKER_SHORTIDS') != '0'
822. shortIdBudget = env('TRACKER_SHORTID_BUDGET_MS') != null ? toNumber(env('TRACKER_SHORTID_BUDGET_MS')) : 40
823. shortIdLogEvery = env('TRACKER_SHORTID_LOG_EVERY') != null ? toNumber(env('TRACKER_SHORTID_LOG_EVERY')) : 5000
824. shortIdRun = { at = 0 batches = 0 assigned = 0 nextLog = 0 slowest = 0 slowestPersist = 0 done = false }
825. shortIdTick = () => {
826. if (shortIdRun.done) { return null }
827. let t = now()
828. let r = shortIdStep(shortIdBudget)
829. let took = now() - t
830. shortIdRun.batches = shortIdRun.batches + 1
831. shortIdRun.assigned = shortIdRun.assigned + r.assigned
832. if (took > shortIdRun.slowest) { shortIdRun.slowest = took }
833. if (r.done || shortIdRun.batches % 20 == 0) {
834. let tp = now()
835. persistShortIds()
836. if (now() - tp > shortIdRun.slowestPersist) { shortIdRun.slowestPersist = now() - tp }
837. }
838. if (!r.done && shortIdRun.assigned >= shortIdRun.nextLog) {
839. console.log('short ids: ' + shortIdRun.assigned + ' assigned, ' + r.left + ' left, ' + ((now() - shortIdRun.at) / 1000) + ' s')
840. shortIdRun.nextLog = shortIdRun.nextLog + shortIdLogEvery
841. }
842. if (r.done) {
843. shortIdRun.done = true
844. let s = shortIdStats()
845. console.log('short ids done: assigned=' + shortIdRun.assigned + ' oldKept=' + s.oldKept + ' batches=' + shortIdRun.batches + ' slowestBatchMs=' + shortIdRun.slowest + ' slowestPersistMs=' + shortIdRun.slowestPersist + ' seconds=' + ((now() - shortIdRun.at) / 1000))
846. }
847. return null
848. }
849. if (!shortIdOn) { console.log('short ids: backfill off (TRACKER_SHORTIDS=0), ' + shortIdsLeft() + ' records without one') }
850. else if (shortIdsLeft() == 0) { console.log('short ids: nothing to do') }
851. else {
852. shortIdRun.at = now()
853. shortIdRun.nextLog = shortIdLogEvery
854. console.log('short ids: backfill start, ' + shortIdsLeft() + ' records, ' + shortIdBudget + ' ms per batch')
855. shortIdClock = every(0.1)
856. on shortIdClock.tick(x) {
857. shortIdTick()
858. return null
859. }
860. }
862. // ---- THE COLLECTION SEED (mission 066, tracker.worldapi.org#19, franchises.hl seedStep) ------------------------------------
863. // TMDB's movie collections become timelines: ONE movie per step (its details; once per collection the collection's parts), on
864. // its own 0.1 s clock like the adult backfill — a pause of 260 ms per TMDB request made (≤ 40 / 10 s). It WAITS while the daily
865. // sync or the adult backfill runs (they share TMDB's limit and the backfill brings the adult flags it skips by); since the merge
866. // (mission 070) also until the details repair, the kind backfill and the credits job are done (or paused) — the LAST job. The queue =
867. // every non-adult movie without a check row, so it RESUMES after a restart and does nothing once all are checked. Persisted
868. // every TRACKER_COLLECTIONS_PERSIST_EVERY (25) steps and at the end. Log: start, every TRACKER_COLLECTIONS_LOG_EVERY (500) movies, done.
869. // TRACKER_COLLECTIONS=0 switches it off; without TMDB_READ_TOKEN it is off too.
870. collectionsOn = env('TRACKER_COLLECTIONS') != '0' && syncEnabled
871. collectionsLogEvery = env('TRACKER_COLLECTIONS_LOG_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_LOG_EVERY')) : 500
872. collectionsPersistEvery = env('TRACKER_COLLECTIONS_PERSIST_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_PERSIST_EVERY')) : 25
873. collectionRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
874. collectionTick = () => {
875. if (collectionRun.done) { return null }
876. // mission 070 (merge): also behind the details repair, the kind backfill and the credits job — one TMDB pacing at a time
877. if (!mergeRun.done || syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && !repairDone) || (kindsOn && !kindDone) || (creditsOn && !creditsDone)) { return null }
878. let t = now()
879. if (!collectionRun.started) {
880. collectionRun.started = true
881. collectionRun.at = t
882. let n = seedPrepare()
883. console.log('collections: start, ' + n + ' movies to check (' + (now() - t) + ' ms to find them)')
884. if (n == 0) { collectionRun.done = true console.log('collections: nothing to do') }
885. return null
886. }
887. if (t < collectionRun.nextAt) { return null }
888. let r = seedStep()
889. if (r.done) {
890. collectionRun.done = true
891. seedPersist()
892. console.log('collections done: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000) + ' slowestStepMs=' + collectionRun.slowest)
893. return null
894. }
895. collectionRun.steps = collectionRun.steps + 1
896. let took = now() - t
897. if (took > collectionRun.slowest) { collectionRun.slowest = took }
898. if (r.error != '') { console.log('collections: ' + r.error) }
899. if (took > 1500) { console.log('collections: slow step, ' + took + ' ms') }
900. collectionRun.nextAt = now() + r.requests * msPerRequest
901. if (collectionRun.steps % collectionsPersistEvery == 0) { seedPersist() }
902. if (collectionRun.steps % collectionsLogEvery == 0) { console.log('collections: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000)) }
903. return null
904. }
905. if (!collectionsOn) { console.log('collections: off (' + (syncEnabled ? 'TRACKER_COLLECTIONS=0' : 'no TMDB_READ_TOKEN') + ')') }
906. else {
907. collectionClock = every(0.1)
908. on collectionClock.tick(x) {
909. collectionTick()
910. return null
911. }
912. }
914. on Error(e) { console.log('error absorbed: ' + e.message) }