tracker

39.8 KB

1. // lib/jobs.hl — THE BACKGROUND JOBS (mission 028: out of project.hl, unchanged): the daily TMDB sync run, the adult and kind
2. // backfills, the details repair, the credits job, the duplicate merge, the short-id backfill and the collection seed — each with
3. // its queue, its pause, its log lines, and the clocks that drive them (which job runs when, which one waits for which) — and, last,
4. // the curated franchise seed (tracker#40). The work of ONE step is the topic's own (sync.hl, backfill.hl, details.hl, credits.hl,
5. // merge.hl, shortids.hl, collections.hl, franchiseseed.hl).
6. // A CLASS, not statics: its state (queues, positions, totals) is rewritten on every step, and a static cannot be reassigned —
7. // project.hl makes the one instance (`jobs = new Jobs()`) after the server, as these lines ran there before; constructing it
8. // starts the clocks and logs which jobs are off.
9. import { env } from 'hl:proc'
10. import { readFile, exists } from 'hl:fs'
11. import { now, timestamp, every } from 'hl:time'
12. import { syncEnabled, pauseMsAfter, msPerRequest } from './tmdb.hl'
13. import { persistSync } from './sync.hl'
14. import { emptyTotals, addTotals, totalsLine } from './sync-helpers.hl'
15. import { planStart, planStep, planIds, planLine, markSynced, syncRunStep } from './dailysync.hl'
16. import { adultBackfillQueue, backfillTitle, kindBackfillQueue, kindTitle, tvmazeStatusQueue, tvmazeStatusTitle } from './backfill.hl'
17. import { detailsRepairQueue, completeStep, persistDetails, applySynced } from './details.hl'
18. import { persistCredits, creditsQueue, creditsStep } from './credits.hl'
19. import { allFollowedShowIds } from './follows.hl'
20. import { refreshCatalogShow } from './catalog.hl'
21. import { searchTitleChanged, searchTitleRenamed } from './search.hl'
22. import { mergePrepare, mergeStep, mergeStats, mergeLine, persistMerge } from './merge.hl'
23. import { shortIdStep, shortIdStats, persistShortIds, shortIdsLeft } from './shortids.hl'
24. import { seedPrepare, seedStep, seedLine, seedPersist } from './collections.hl'
25. import { franchiseSeedPrepare, franchiseSeedStep, franchiseSeedLine, franchiseSeedStats } from './franchiseseed.hl'
27. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, sync.hl; tracker#18, dailysync.hl) -------------------------
28. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC). hl:web serves ONE
29. // request at a time, so the run never takes the server for itself: a 1-second clock does ONE step per tick and returns — the
30. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
31. // tracker#18: the run first asks TMDB's and TVmaze's CHANGE LISTS since the last run (dailysync.hl — one list page per step)
32. // and then syncs only the titles of ours that changed, followed ones first — every title stays current, not only the followed
33. // ones. FULL WALK (every followed show, as before #18) on the first run, after a gap > 14 days, or when a change list fails.
34. // A title's step: sync.hl syncShow (1–3 TMDB requests + maybe its poster, TVmaze) + details.hl applySynced (the record's
35. // title, overview, tagline … and cast/crew from the same details answer). After a step that made n requests the next waits
36. // n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze request (≤ 20 / 10 s, tracker.worldapi.org#12).
37. // The finished run stores its day (dailysync.hl markSynced). Log: "tmdb sync: start, …", the change lists ("tmdb sync:
38. // changes since …"), errors, slow steps, "tmdb sync done: mode=… changeRequests=… <totals>".
39. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
40. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
41. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
42. syncQueue = []
43. syncPlanning = false
44. syncMode = ''
45. syncMark = false
46. syncChangeRequests = 0
47. syncAt = 0
48. syncDay = ''
49. syncNextAt = 0
50. syncTotals = emptyTotals()
51. syncFinish = () => {
52. persistDetails()
53. if (syncMark) { markSynced(syncDay) }
54. console.log('tmdb sync done: mode=' + syncMode + ' changeRequests=' + syncChangeRequests + ' ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000))
55. return null
56. }
57. syncTick = () => {
58. t = now()
59. if (syncQueue.length == 0 && !syncPlanning) {
60. stamp = timestamp(t)
61. day = stamp.slice(0, 10)
62. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
63. syncDay = day
64. syncAt = t
65. syncNextAt = 0
66. syncTotals = emptyTotals()
67. syncChangeRequests = 0
68. p = planStart(day, null)
69. syncMode = p.mode
70. syncMark = true
71. if (p.mode == 'delta') {
72. syncPlanning = true
73. console.log('tmdb sync: start, changes ' + p.why)
74. return null
75. }
76. syncQueue = allFollowedShowIds()
77. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows, full walk (' + p.why + ')')
78. if (syncQueue.length == 0) { syncFinish() }
79. return null
80. }
81. if (t < syncNextAt) { return null }
82. if (syncPlanning) {
83. ps = planStep()
84. syncChangeRequests = syncChangeRequests + ps.requests + ps.tvmazeRequests
85. syncTotals.requests = syncTotals.requests + ps.requests
86. syncTotals.tvmazeRequests = syncTotals.tvmazeRequests + ps.tvmazeRequests
87. syncNextAt = now() + pauseMsAfter(ps)
88. if (!ps.done) { return null }
89. syncPlanning = false
90. console.log('tmdb sync: ' + planLine())
91. if (ps.failed != '') {
92. // a change list could not be read: the old full walk, and the day is not advanced (the next run asks again)
93. syncMode = 'full'
94. syncMark = false
95. syncQueue = allFollowedShowIds()
96. console.log('tmdb sync: change list failed, full walk of ' + syncQueue.length + ' followed shows instead')
97. } else {
98. syncQueue = planIds()
99. }
100. if (syncQueue.length == 0) { syncFinish() }
101. return null
102. }
103. id = syncQueue[0]
104. rest = []
105. let i = 1
106. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
107. syncQueue = rest
108. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze); tracker#35: one only on
109. // TVmaze's change list its TVmaze status (dailysync.hl syncRunStep)
110. r = syncRunStep(id, syncMode == 'delta')
111. // tracker#18 + #28: the record's own fields (title, overview, tagline …) and the cast + crew from the same details answer
112. if (r.details != null) {
113. c = applySynced(id, r.details)
114. if (c.error != '') { console.log('tmdb sync: ' + c.error) }
115. if (c.renamed) { searchTitleRenamed(id) }
116. }
117. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
118. // also brings TMDB's adult flag (mission 054) → the search's view of it too
119. refreshCatalogShow(id)
120. searchTitleChanged(id)
121. took = now() - t
122. syncTotals = addTotals(syncTotals, r)
123. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
124. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
125. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
126. syncNextAt = now() + pauseMsAfter(r)
127. if (syncQueue.length > 0 && syncTotals.shows % 10 == 0) { persistDetails() }
128. if (syncQueue.length == 0) { syncFinish() }
129. return null
130. }
132. // ---- THE ADULT-FLAG BACKFILL (mission 054, backfill.hl backfillTitle) -----------------------------------
133. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
134. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
135. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
136. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
137. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
138. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
139. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
140. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
141. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
142. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
143. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
144. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
145. backfillQueue = null
146. backfillPos = 0
147. backfillDone = false
148. backfillAt = 0
149. backfillNextAt = 0
150. backfillTries = {}
151. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
152. backfillLine = () => {
153. b = backfillTotals
154. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
155. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
156. }
157. backfillTick = () => {
158. if (backfillDone) { return null }
159. t = now()
160. if (backfillQueue == null) {
161. backfillQueue = adultBackfillQueue()
162. backfillAt = t
163. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
164. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
165. return null
166. }
167. if (t < backfillNextAt) { return null }
168. id = backfillQueue[backfillPos]
169. backfillPos = backfillPos + 1
170. r = backfillTitle(id)
171. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
172. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
173. q = backfillQueue
174. q.push(id)
175. backfillQueue = q
176. backfillTotals.retries = backfillTotals.retries + 1
177. } else {
178. backfillTotals.titles = backfillTotals.titles + 1
179. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
180. }
181. backfillTotals.posters = backfillTotals.posters + r.posters
182. backfillTotals.ids = backfillTotals.ids + r.ids
183. backfillTotals.requests = backfillTotals.requests + r.requests
184. backfillTotals.errors = backfillTotals.errors + r.errors
185. if (r.error != '') { console.log('adult backfill: ' + r.error) }
186. if (r.adult != null) {
187. refreshCatalogShow(id)
188. searchTitleChanged(id)
189. }
190. took = now() - t
191. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
192. backfillNextAt = now() + r.requests * msPerRequest
193. finished = backfillPos >= backfillQueue.length
194. if (finished || backfillPos % 25 == 0) { persistSync() }
195. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
196. if (finished) {
197. backfillDone = true
198. console.log('adult backfill done: ' + backfillLine())
199. }
200. return null
201. }
203. // ---- THE DETAILS REPAIR (tracker.worldapi.org#26, details.hl completeStep) ----------------------------------------
204. // Titles the person pages added from a filmography (#16) are MINIMAL (no poster, seasons, ids, cast …), and (tracker#33, mission
205. // 029) the old tracker's migrated rows were never synced (5,697 on the live copy, 691 series without seasons) — shows.hl
206. // isIncomplete. Opening one completes it (components/show.hl); this job completes the rest in the background: at start it
207. // collects every incomplete, non-adult title (the visibly reduced ones first, details.hl detailsRepairQueue) and does ONE per
208. // step on the app's clock, like the adult backfill — a step is the daily sync's work for one title
209. // (1–3 TMDB requests + poster + TVmaze), requests queued meanwhile are served between two steps; then a pause of 260 ms per
210. // TMDB request + 550 ms per TVmaze request (pauseMsAfter: ≤ 40 / 10 s, ≤ 20 / 10 s). RESUMES after a restart (the queue is
211. // what is still incomplete); TMDB 404 marks the title (`detailsCheck`), a passing failure goes to the back of the queue (at
212. // most 2 more tries per start). Waits while the daily sync or the adult backfill runs. Persisted every 10 titles + at the end.
213. // Log: start, every TRACKER_REPAIR_LOG_EVERY (100) titles, each failure, done. TRACKER_REPAIR=0 switches it off (off too
214. // without TMDB_READ_TOKEN). TRACKER_REPAIR_FAST=1 (the gate, fake TMDB only): 20 ms pause instead.
215. repairOn = env('TRACKER_REPAIR') != '0' && syncEnabled
216. repairFast = env('TRACKER_REPAIR_FAST') == '1'
217. repairLogEvery = env('TRACKER_REPAIR_LOG_EVERY') != null ? toNumber(env('TRACKER_REPAIR_LOG_EVERY')) : 100
218. repairQueue = null
219. repairPos = 0
220. repairDone = false
221. repairAt = 0
222. repairNextAt = 0
223. repairTries = {}
224. repairTotals = { titles = 0 complete = 0 failed = 0 adult = 0 newSeasons = 0 newEpisodes = 0 posters = 0 cast = 0 requests = 0 tvmazeRequests = 0 retries = 0 }
225. repairLine = () => {
226. b = repairTotals
227. return b.titles + '/' + repairQueue.length + ' titles, complete=' + b.complete + ' failed=' + b.failed + ' adult=' + b.adult + ' newSeasons=' + b.newSeasons + ' newEpisodes=' + b.newEpisodes
228. + ' posters=' + b.posters + ' cast=' + b.cast + ' requests=' + b.requests + ' tvmazeRequests=' + b.tvmazeRequests + ' retries=' + b.retries + ' seconds=' + ((now() - repairAt) / 1000)
229. }
230. repairTick = () => {
231. if (repairDone) { return null }
232. t = now()
233. if (repairQueue == null) {
234. repairQueue = detailsRepairQueue()
235. repairAt = t
236. console.log('details repair: start, ' + repairQueue.length + ' incomplete titles (' + (now() - t) + ' ms to find them)')
237. if (repairQueue.length == 0) { repairDone = true console.log('details repair: nothing to do') }
238. return null
239. }
240. if (t < repairNextAt) { return null }
241. id = repairQueue[repairPos]
242. r = completeStep(id)
243. took = now() - t
244. if (took > 1500) { console.log('details repair: slow step, show ' + id + ', ' + took + ' ms') }
245. if (r.error != '') { console.log('details repair: ' + r.error) }
246. repairNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
247. // a series takes several steps: the same title again next time
248. if (!r.done && !r.retry) { return null }
249. repairPos = repairPos + 1
250. if (r.retry && (repairTries[id] == null || repairTries[id] < 2)) {
251. repairTries[id] = repairTries[id] == null ? 1 : repairTries[id] + 1
252. q = repairQueue
253. q.push(id)
254. repairQueue = q
255. repairTotals.retries = repairTotals.retries + 1
256. } else if (!r.skipped) {
257. // done: its counts (a failed first step: the request alone)
258. sum = r.sum
259. repairTotals.titles = repairTotals.titles + 1
260. if (r.adult == true) { repairTotals.adult = repairTotals.adult + 1 } else if (sum.steps == 0) { repairTotals.failed = repairTotals.failed + 1 } else { repairTotals.complete = repairTotals.complete + 1 }
261. repairTotals.newSeasons = repairTotals.newSeasons + sum.newSeasons
262. repairTotals.newEpisodes = repairTotals.newEpisodes + sum.newEpisodes
263. repairTotals.posters = repairTotals.posters + sum.posters
264. repairTotals.cast = repairTotals.cast + sum.cast
265. repairTotals.requests = repairTotals.requests + (sum.steps == 0 ? r.requests : sum.requests)
266. repairTotals.tvmazeRequests = repairTotals.tvmazeRequests + sum.tvmazeRequests
267. }
268. finished = repairPos >= repairQueue.length
269. if (finished || repairPos % 10 == 0) { persistDetails() }
270. if (!finished && !r.skipped && !r.retry && repairTotals.titles > 0 && repairTotals.titles % repairLogEvery == 0) { console.log('details repair: ' + repairLine()) }
271. if (finished) {
272. repairDone = true
273. console.log('details repair done: ' + repairLine())
274. }
275. return null
276. }
278. // ---- THE TVMAZE STATUS BACKFILL (tracker#35, mission 030, backfill.hl tvmazeStatusTitle) — the details repair's second part ----
279. // The page's status pill shows TVmaze's status when a series has one (fresher than TMDB's). The sync stores it whenever it merges
280. // TVmaze's episodes (followed series, every completion); the series it never reached get it here: once the details repair is
281. // done (same switch, TRACKER_REPAIR) every series with a TVmaze id and no tvmazeStatus, ONE TVmaze request per step, then 550 ms
282. // (TVmaze: ≤ 20 / 10 s). RESUMES after a restart (the queue = what still has none); a 404 marks the series
283. // (`tvmazeStatusCheck`), a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily
284. // sync runs. Persisted every 25 series + at the end. Log: start, every TRACKER_REPAIR_LOG_EVERY series, done.
285. statusQueue = null
286. statusPos = 0
287. statusDone = false
288. statusAt = 0
289. statusNextAt = 0
290. statusTries = {}
291. statusTotals = { series = 0 stored = 0 notFound = 0 requests = 0 errors = 0 retries = 0 }
292. statusLine = () => {
293. b = statusTotals
294. return b.series + '/' + statusQueue.length + ' series, stored=' + b.stored + ' notFound=' + b.notFound + ' tvmazeRequests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - statusAt) / 1000)
295. }
296. statusTick = () => {
297. if (statusDone) { return null }
298. t = now()
299. if (statusQueue == null) {
300. statusQueue = tvmazeStatusQueue()
301. statusAt = t
302. console.log('tvmaze status: start, ' + statusQueue.length + ' series without a TVmaze status (' + (now() - t) + ' ms to find them)')
303. if (statusQueue.length == 0) { statusDone = true console.log('tvmaze status: nothing to do') }
304. return null
305. }
306. if (t < statusNextAt) { return null }
307. id = statusQueue[statusPos]
308. statusPos = statusPos + 1
309. r = tvmazeStatusTitle(id)
310. if (r.retry && (statusTries[id] == null || statusTries[id] < 2)) {
311. statusTries[id] = statusTries[id] == null ? 1 : statusTries[id] + 1
312. q = statusQueue
313. q.push(id)
314. statusQueue = q
315. statusTotals.retries = statusTotals.retries + 1
316. } else {
317. statusTotals.series = statusTotals.series + 1
318. if (r.status != null) { statusTotals.stored = statusTotals.stored + 1 } else if (r.errors > 0) { statusTotals.notFound = statusTotals.notFound + 1 }
319. }
320. statusTotals.requests = statusTotals.requests + r.tvmazeRequests
321. statusTotals.errors = statusTotals.errors + r.errors
322. if (r.error != '') { console.log('tvmaze status: ' + r.error) }
323. statusNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
324. finished = statusPos >= statusQueue.length
325. if (finished || statusPos % 25 == 0) { persistSync() }
326. if (!finished && !r.retry && statusTotals.series > 0 && statusTotals.series % repairLogEvery == 0) { console.log('tvmaze status: ' + statusLine()) }
327. if (finished) {
328. statusDone = true
329. console.log('tvmaze status done: ' + statusLine())
330. }
331. return null
332. }
334. // ---- THE KIND BACKFILL (tracker.worldapi.org#21, mission 068, backfill.hl kindTitle) ----------------------------------
335. // Every TV title without a stored kind gets one: a title with a show genre (Reality / Talk / News) from the stored data (no
336. // request), any other from TMDB's TV type (ONE request: GET /tv/<id>, then 260 ms pause). Starts once the adult backfill is
337. // done (or off), waits while the daily sync runs, ONE title per step on the same clock. A title that becomes a SHOW moves from
338. // /series to /shows at once (catalog.hl refreshCatalogShow). RESUMES after a restart (the queue = every TV title still without
339. // a kind); a 404 stores the genres' answer + `kindCheck`; a passing failure goes to the back of the queue, at most 2 more tries
340. // per start. Log: start, every TRACKER_KINDS_LOG_EVERY (500) titles, done. TRACKER_KINDS=0 switches it off; without
341. // TMDB_READ_TOKEN it is off too (the lists still split by genre: shows.hl titleKind).
342. kindsOn = env('TRACKER_KINDS') != '0' && syncEnabled
343. kindsLogEvery = env('TRACKER_KINDS_LOG_EVERY') != null ? toNumber(env('TRACKER_KINDS_LOG_EVERY')) : 500
344. kindQueue = null
345. kindPos = 0
346. kindDone = false
347. kindAt = 0
348. kindNextAt = 0
349. kindTries = {}
350. kindTotals = { titles = 0 series = 0 shows = 0 fromGenre = 0 moved = 0 requests = 0 errors = 0 retries = 0 }
351. kindLine = () => {
352. k = kindTotals
353. return k.titles + '/' + kindQueue.length + ' titles, series=' + k.series + ' shows=' + k.shows + ' fromGenre=' + k.fromGenre + ' moved=' + k.moved
354. + ' requests=' + k.requests + ' errors=' + k.errors + ' retries=' + k.retries + ' seconds=' + ((now() - kindAt) / 1000)
355. }
356. kindTick = () => {
357. if (kindDone) { return null }
358. t = now()
359. if (kindQueue == null) {
360. kindQueue = kindBackfillQueue()
361. kindAt = t
362. console.log('kind backfill: start, ' + kindQueue.length + ' TV titles without a kind (' + (now() - t) + ' ms to find them)')
363. if (kindQueue.length == 0) { kindDone = true console.log('kind backfill: nothing to do') }
364. return null
365. }
366. if (t < kindNextAt) { return null }
367. id = kindQueue[kindPos]
368. kindPos = kindPos + 1
369. r = kindTitle(id)
370. if (r.retry && (kindTries[id] == null || kindTries[id] < 2)) {
371. kindTries[id] = kindTries[id] == null ? 1 : kindTries[id] + 1
372. q = kindQueue
373. q.push(id)
374. kindQueue = q
375. kindTotals.retries = kindTotals.retries + 1
376. } else {
377. kindTotals.titles = kindTotals.titles + 1
378. if (r.kind == 'show') { kindTotals.shows = kindTotals.shows + 1 } else if (r.kind == 'series') { kindTotals.series = kindTotals.series + 1 }
379. if (r.fromGenre) { kindTotals.fromGenre = kindTotals.fromGenre + 1 }
380. }
381. kindTotals.requests = kindTotals.requests + r.requests
382. kindTotals.errors = kindTotals.errors + r.errors
383. if (r.error != '') { console.log('kind backfill: ' + r.error) }
384. // until now the genres decided (no show genre = series): only a title TMDB's type makes a SHOW changes its list
385. if (r.kind == 'show' && !r.fromGenre) {
386. refreshCatalogShow(id)
387. kindTotals.moved = kindTotals.moved + 1
388. }
389. took = now() - t
390. if (took > 1500) { console.log('kind backfill: slow step, show ' + id + ', ' + took + ' ms') }
391. kindNextAt = now() + r.requests * msPerRequest
392. finished = kindPos >= kindQueue.length
393. if (finished || kindPos % 25 == 0) { persistSync() }
394. if (!finished && kindTotals.titles > 0 && kindTotals.titles % kindsLogEvery == 0 && !r.retry) { console.log('kind backfill: ' + kindLine()) }
395. if (finished) {
396. kindDone = true
397. console.log('kind backfill done: ' + kindLine())
398. }
399. return null
400. }
402. // ---- THE CREDITS JOB (tracker.worldapi.org#28, credits.hl creditsStep) ------------------------------------------------------
403. // Every PUBLIC title (adult == false; adult/unknown are skipped) with a TMDB id gets its whole cast + crew once: at start (after
404. // the adult backfill and the details repair are done) the job collects every such title without `creditsAt` and asks ONE per
405. // step on the app's clock — one request (details + credits / aggregate_credits), then 260 ms (≤ 40 / 10 s); requests queued
406. // meanwhile are served between two steps. Titles the details repair, the daily sync or an import already did carry `creditsAt`
407. // and are not asked. RESUMES after a restart (the queue = what still has no credits); TMDB 404 / adult → `creditsCheck`, never
408. // asked again; a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily sync
409. // runs. Persisted (shows + people only) every 25 titles + at the end. Log: start, every TRACKER_CREDITS_LOG_EVERY (500) titles, failures, slow steps
410. // (> 1.5 s), done. TRACKER_CREDITS=0 switches it off (off too without TMDB_READ_TOKEN). TRACKER_CREDITS_FAST=1 (the gate, fake
411. // TMDB only): 20 ms pause instead.
412. // MEMORY LIMIT: the runtime never frees what a fetch + JSON parse took (mission 061 measured ~6 MB more RSS per title on the real
413. // data, 300 fetches of a 440 KB JSON = +2.7 GB in a 10-line script) — the whole job in one process would add ~35 GB. Before every
414. // step the job reads its own RSS (/proc/self/status); above TRACKER_CREDITS_MAX_MB (16000) it PAUSES for this process (logged) and
415. // goes on with what is left after the next restart (deploy, `docker restart`).
416. creditsOn = env('TRACKER_CREDITS') != '0' && syncEnabled
417. creditsFast = env('TRACKER_CREDITS_FAST') == '1'
418. creditsLogEvery = env('TRACKER_CREDITS_LOG_EVERY') != null ? toNumber(env('TRACKER_CREDITS_LOG_EVERY')) : 500
419. creditsQueueIds = null
420. creditsPos = 0
421. creditsDone = false
422. creditsAt = 0
423. creditsNextAt = 0
424. creditsTries = {}
425. creditsMaxMb = env('TRACKER_CREDITS_MAX_MB') != null ? toNumber(env('TRACKER_CREDITS_MAX_MB')) : 16000
426. creditsTotals = { titles = 0 done = 0 failed = 0 cast = 0 crew = 0 people = 0 requests = 0 retries = 0 slowest = 0 }
427. // this process' resident memory in MB (VmRSS), 0 when unknown
428. rssMb = () => {
429. if (!exists('/proc/self/status')) { return 0 }
430. t = readFile('/proc/self/status')
431. i = t != null ? t.indexOf('VmRSS:') : -1
432. if (i < 0) { return 0 }
433. kb = toNumber(t.slice(i + 6, t.indexOf('kB', i)).trim())
434. return kb != null ? (kb - kb % 1024) / 1024 : 0
435. }
436. creditsLine = () => {
437. b = creditsTotals
438. return b.titles + '/' + creditsQueueIds.length + ' titles, done=' + b.done + ' failed=' + b.failed + ' cast=' + b.cast + ' crew=' + b.crew + ' people=' + b.people
439. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - creditsAt) / 1000)
440. }
441. creditsTick = () => {
442. if (creditsDone) { return null }
443. t = now()
444. if (creditsQueueIds == null) {
445. creditsQueueIds = creditsQueue()
446. creditsAt = t
447. console.log('credits job: start, ' + creditsQueueIds.length + ' titles without credits (' + (now() - t) + ' ms to find them)')
448. if (creditsQueueIds.length == 0) { creditsDone = true console.log('credits job: nothing to do') }
449. return null
450. }
451. if (t < creditsNextAt) { return null }
452. mb = rssMb()
453. if (mb > creditsMaxMb) {
454. creditsDone = true
455. persistCredits()
456. console.log('credits job: paused, RSS ' + mb + ' MB > TRACKER_CREDITS_MAX_MB ' + creditsMaxMb + ' — ' + (creditsQueueIds.length - creditsPos) + ' titles left for the next start; ' + creditsLine())
457. return null
458. }
459. id = creditsQueueIds[creditsPos]
460. creditsPos = creditsPos + 1
461. r = creditsStep(id)
462. took = now() - t
463. if (took > creditsTotals.slowest) { creditsTotals.slowest = took }
464. if (took > 1500) { console.log('credits job: slow step, show ' + id + ', ' + took + ' ms, cast=' + r.cast + ' crew=' + r.crew + ' people=' + r.people) }
465. if (r.error != '') { console.log('credits job: ' + r.error) }
466. creditsNextAt = now() + (creditsFast ? 20 : r.requests * msPerRequest)
467. if (r.retry && (creditsTries[id] == null || creditsTries[id] < 2)) {
468. creditsTries[id] = creditsTries[id] == null ? 1 : creditsTries[id] + 1
469. q = creditsQueueIds
470. q.push(id)
471. creditsQueueIds = q
472. creditsTotals.retries = creditsTotals.retries + 1
473. } else if (!r.skipped) {
474. creditsTotals.titles = creditsTotals.titles + 1
475. if (r.error != '') { creditsTotals.failed = creditsTotals.failed + 1 } else { creditsTotals.done = creditsTotals.done + 1 }
476. creditsTotals.cast = creditsTotals.cast + r.cast
477. creditsTotals.crew = creditsTotals.crew + r.crew
478. creditsTotals.people = creditsTotals.people + r.people
479. }
480. creditsTotals.requests = creditsTotals.requests + r.requests
481. finished = creditsPos >= creditsQueueIds.length
482. if (finished || creditsPos % 25 == 0) { persistCredits() }
483. if (!finished && !r.skipped && !r.retry && creditsTotals.titles > 0 && creditsTotals.titles % creditsLogEvery == 0) { console.log('credits job: ' + creditsLine()) }
484. if (finished) {
485. creditsDone = true
486. console.log('credits job done: ' + creditsLine())
487. }
488. return null
489. }
491. // ---- THE DUPLICATE MERGE (tracker.worldapi.org#31, mission 026, merge.hl) ----------------------------------------------------
492. // The old tracker's duplicate titles (same type + TMDB id) become ONE record each — follows, watches, seasons, cast, credits and
493. // timelines moved to the keeper, the others kept as tombstones whose slug and short id lead to it — and seasons that name a title
494. // without being in its list are merged into their twin or linked (merge.hl has the rules). On its own clock (every 0.1 s) ONE
495. // item per tick (a duplicate group, or one title's stray seasons; no TMDB), so pages are served between two items. The TMDB jobs
496. // below (daily sync, backfills, repair, credits, collections) wait until it is done: none of them works on a record that is about
497. // to become a tombstone. RESUMES after a restart (the queue = what is still duplicated / stray). Persisted every 10 items and at
498. // the end. Log: start, one line per tombstone / repaired title, done; with nothing to do one line. TRACKER_MERGE=0 switches it off.
499. mergeOn = env('TRACKER_MERGE') != '0'
500. mergeRun = { started = false done = !mergeOn at = 0 items = 0 slowest = 0 }
501. mergeTick = () => {
502. if (mergeRun.done) { return null }
503. t = now()
504. if (!mergeRun.started) {
505. mergeRun.started = true
506. mergeRun.at = t
507. n = mergePrepare()
508. st = mergeStats()
509. console.log('merge: start, ' + st.groups + ' duplicate groups (' + st.records + ' records), ' + st.strayTitles + ' titles with stray seasons (' + st.ms + ' ms to find them)')
510. if (n == 0) {
511. mergeRun.done = true
512. console.log('merge: nothing to do')
513. }
514. return null
515. }
516. r = mergeStep()
517. mergeRun.items = mergeRun.items + 1
518. took = now() - t
519. if (took > mergeRun.slowest) { mergeRun.slowest = took }
520. if (took > 1500) { console.log('merge: slow step, ' + took + ' ms') }
521. if (r.done || mergeRun.items % 10 == 0) { persistMerge() }
522. if (r.done) {
523. mergeRun.done = true
524. console.log('merge done: ' + mergeLine() + ' items=' + mergeRun.items + ' slowestStepMs=' + mergeRun.slowest + ' seconds=' + ((now() - mergeRun.at) / 1000))
525. }
526. return null
527. }
528. if (!mergeOn) { console.log('merge: off (TRACKER_MERGE=0)') }
529. else {
530. mergeClock = every(0.1)
531. on mergeClock.tick(x) {
532. mergeTick()
533. return null
534. }
535. }
537. // ONE clock for all five (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in
538. // between, only while no daily run is going on; the details repair only when neither has work (then its TVmaze status part,
539. // tracker#35); then the kind backfill (#21);
540. // the credits job (#28) last
541. clockOn = syncOn || backfillOn || repairOn || kindsOn || creditsOn
542. clockTicks = 0
543. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
544. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
545. if (!repairOn) { console.log('details repair: off (' + (syncEnabled ? 'TRACKER_REPAIR=0' : 'no TMDB_READ_TOKEN') + ')') }
546. if (!creditsOn) { console.log('credits job: off (' + (syncEnabled ? 'TRACKER_CREDITS=0' : 'no TMDB_READ_TOKEN') + ')') }
547. if (!kindsOn) { console.log('kind backfill: off (' + (syncEnabled ? 'TRACKER_KINDS=0' : 'no TMDB_READ_TOKEN') + ')') }
548. if (clockOn) {
549. syncClock = every(0.1)
550. on syncClock.tick(x) {
551. clockTicks = clockTicks + 1
552. // tracker#31: not before the duplicate merge is done
553. if (!mergeRun.done) { return null }
554. busy = syncQueue.length > 0 || syncPlanning
555. if (syncOn && clockTicks % 10 == 0) { syncTick() }
556. free = !busy && syncQueue.length == 0 && !syncPlanning
557. if (backfillOn && free && !backfillDone) { backfillTick() }
558. else if (repairOn && free && (!backfillOn || backfillDone) && !repairDone) { repairTick() }
559. else if (repairOn && free && (!backfillOn || backfillDone) && !statusDone) { statusTick() }
560. else if (kindsOn && free && (!backfillOn || backfillDone) && (!repairOn || (repairDone && statusDone)) && !kindDone) { kindTick() }
561. else if (creditsOn && free && (!backfillOn || backfillDone) && (!repairOn || (repairDone && statusDone)) && (!kindsOn || kindDone)) { creditsTick() }
562. return null
563. }
564. }
566. // ---- THE SHORT-ID BACKFILL (mission 060, tracker.worldapi.org#27, shortids.hl shortIdStep) ----------------------------
567. // Every show and person without a short id gets one in the background: on its own clock (every 0.1 s) ONE batch per tick,
568. // as many records as fit into TRACKER_SHORTID_BUDGET_MS (default 40 ms) — no TMDB, only the tables — so a page waits at
569. // most one batch. The queue is built at start (`ensureShortIds`) from what is still missing = it resumes after a restart.
570. // Persisted every 20 batches and at the end. Log: start, every TRACKER_SHORTID_LOG_EVERY (5000) records, done; with nothing
571. // to do one line. TRACKER_SHORTIDS=0 switches it off (the gate's resume check).
572. shortIdOn = env('TRACKER_SHORTIDS') != '0'
573. shortIdBudget = env('TRACKER_SHORTID_BUDGET_MS') != null ? toNumber(env('TRACKER_SHORTID_BUDGET_MS')) : 40
574. shortIdLogEvery = env('TRACKER_SHORTID_LOG_EVERY') != null ? toNumber(env('TRACKER_SHORTID_LOG_EVERY')) : 5000
575. shortIdRun = { at = 0 batches = 0 assigned = 0 nextLog = 0 slowest = 0 slowestPersist = 0 done = false }
576. shortIdTick = () => {
577. if (shortIdRun.done) { return null }
578. t = now()
579. r = shortIdStep(shortIdBudget)
580. took = now() - t
581. shortIdRun.batches = shortIdRun.batches + 1
582. shortIdRun.assigned = shortIdRun.assigned + r.assigned
583. if (took > shortIdRun.slowest) { shortIdRun.slowest = took }
584. if (r.done || shortIdRun.batches % 20 == 0) {
585. tp = now()
586. persistShortIds()
587. if (now() - tp > shortIdRun.slowestPersist) { shortIdRun.slowestPersist = now() - tp }
588. }
589. if (!r.done && shortIdRun.assigned >= shortIdRun.nextLog) {
590. console.log('short ids: ' + shortIdRun.assigned + ' assigned, ' + r.left + ' left, ' + ((now() - shortIdRun.at) / 1000) + ' s')
591. shortIdRun.nextLog = shortIdRun.nextLog + shortIdLogEvery
592. }
593. if (r.done) {
594. shortIdRun.done = true
595. s = shortIdStats()
596. console.log('short ids done: assigned=' + shortIdRun.assigned + ' oldKept=' + s.oldKept + ' batches=' + shortIdRun.batches + ' slowestBatchMs=' + shortIdRun.slowest + ' slowestPersistMs=' + shortIdRun.slowestPersist + ' seconds=' + ((now() - shortIdRun.at) / 1000))
597. }
598. return null
599. }
600. if (!shortIdOn) { console.log('short ids: backfill off (TRACKER_SHORTIDS=0), ' + shortIdsLeft() + ' records without one') }
601. else if (shortIdsLeft() == 0) { console.log('short ids: nothing to do') }
602. else {
603. shortIdRun.at = now()
604. shortIdRun.nextLog = shortIdLogEvery
605. console.log('short ids: backfill start, ' + shortIdsLeft() + ' records, ' + shortIdBudget + ' ms per batch')
606. shortIdClock = every(0.1)
607. on shortIdClock.tick(x) {
608. shortIdTick()
609. return null
610. }
611. }
613. // ---- THE COLLECTION SEED (mission 066, tracker.worldapi.org#19, collections.hl seedStep) ------------------------------------
614. // TMDB's movie collections become timelines: ONE movie per step (its details; once per collection the collection's parts), on
615. // its own 0.1 s clock like the adult backfill — a pause of 260 ms per TMDB request made (≤ 40 / 10 s). It WAITS while the daily
616. // sync or the adult backfill runs (they share TMDB's limit and the backfill brings the adult flags it skips by); since the merge
617. // (mission 070) also until the details repair, the kind backfill and the credits job are done (or paused) — the LAST job. The queue =
618. // every non-adult movie without a check row, so it RESUMES after a restart and does nothing once all are checked. Persisted
619. // every TRACKER_COLLECTIONS_PERSIST_EVERY (25) steps and at the end. Log: start, every TRACKER_COLLECTIONS_LOG_EVERY (500) movies, done.
620. // TRACKER_COLLECTIONS=0 switches it off; without TMDB_READ_TOKEN it is off too.
621. collectionsOn = env('TRACKER_COLLECTIONS') != '0' && syncEnabled
622. collectionsLogEvery = env('TRACKER_COLLECTIONS_LOG_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_LOG_EVERY')) : 500
623. collectionsPersistEvery = env('TRACKER_COLLECTIONS_PERSIST_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_PERSIST_EVERY')) : 25
624. collectionRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
625. collectionTick = () => {
626. if (collectionRun.done) { return null }
627. // mission 070 (merge): also behind the details repair, the kind backfill and the credits job — one TMDB pacing at a time
628. if (!mergeRun.done || syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && (!repairDone || !statusDone)) || (kindsOn && !kindDone) || (creditsOn && !creditsDone)) { return null }
629. t = now()
630. if (!collectionRun.started) {
631. collectionRun.started = true
632. collectionRun.at = t
633. n = seedPrepare()
634. console.log('collections: start, ' + n + ' movies to check (' + (now() - t) + ' ms to find them)')
635. if (n == 0) { collectionRun.done = true console.log('collections: nothing to do') }
636. return null
637. }
638. if (t < collectionRun.nextAt) { return null }
639. r = seedStep()
640. if (r.done) {
641. collectionRun.done = true
642. seedPersist()
643. console.log('collections done: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000) + ' slowestStepMs=' + collectionRun.slowest)
644. return null
645. }
646. collectionRun.steps = collectionRun.steps + 1
647. took = now() - t
648. if (took > collectionRun.slowest) { collectionRun.slowest = took }
649. if (r.error != '') { console.log('collections: ' + r.error) }
650. if (took > 1500) { console.log('collections: slow step, ' + took + ' ms') }
651. collectionRun.nextAt = now() + r.requests * msPerRequest
652. if (collectionRun.steps % collectionsPersistEvery == 0) { seedPersist() }
653. if (collectionRun.steps % collectionsLogEvery == 0) { console.log('collections: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000)) }
654. return null
655. }
656. if (!collectionsOn) { console.log('collections: off (' + (syncEnabled ? 'TRACKER_COLLECTIONS=0' : 'no TMDB_READ_TOKEN') + ')') }
657. else {
658. collectionClock = every(0.1)
659. on collectionClock.tick(x) {
660. collectionTick()
661. return null
662. }
663. }
665. // ---- THE CURATED FRANCHISE SEED (tracker.worldapi.org#40, mission 034, franchiseseed.hl) --------------------------------------
666. // data/franchises.json → franchises + timelines, movies AND series. The LAST start job: it waits until the merge, the daily sync,
667. // the backfills, the details repair, the kind backfill, the credits job and the collection seed are done (or off) — the
668. // collection seed's timelines are what the file's `collections` attach. Then on its own 0.1 s clock ONE step per tick: the next
669. // missing title imported (the search's import, complete with cast + crew; its log line `search import: …`), then the pause the daily sync makes after the same
670. // requests (≤ 40 TMDB / 10 s, ≤ 20 TVmaze / 10 s); the queue done, the build (no TMDB). RESUMES after a restart (the queue = the
671. // titles still missing), IDEMPOTENT (nothing missing + nothing changed = no write). Without TMDB_READ_TOKEN it only builds from
672. // what we have. Log: start, every import, every TRACKER_FRANCHISE_SEED_LOG_EVERY (25) imports, done.
673. // TRACKER_FRANCHISE_SEED=0 switches it off; TRACKER_FRANCHISE_FILE another data file (the gate's).
674. franchiseSeedOn = env('TRACKER_FRANCHISE_SEED') != '0'
675. franchiseSeedLogEvery = env('TRACKER_FRANCHISE_SEED_LOG_EVERY') != null ? toNumber(env('TRACKER_FRANCHISE_SEED_LOG_EVERY')) : 25
676. franchiseRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
677. franchiseSeedTick = () => {
678. if (franchiseRun.done) { return null }
679. if (!mergeRun.done || syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && (!repairDone || !statusDone)) || (kindsOn && !kindDone) || (creditsOn && !creditsDone) || (collectionsOn && !collectionRun.done)) { return null }
680. t = now()
681. if (!franchiseRun.started) {
682. franchiseRun.started = true
683. franchiseRun.at = t
684. n = franchiseSeedPrepare(syncEnabled)
685. st = franchiseSeedStats()
686. if (!st.file) {
687. franchiseRun.done = true
688. console.log('franchise seed: ' + st.error + ' — nothing seeded')
689. return null
690. }
691. console.log('franchise seed: start, ' + st.franchises + ' franchises, ' + st.timelines + ' timelines, ' + st.titles + ' titles in the file — ' + st.have + ' we have, ' + n + ' to import, ' + st.missing + ' TMDB lacks (' + (now() - t) + ' ms)')
692. return null
693. }
694. if (t < franchiseRun.nextAt) { return null }
695. r = franchiseSeedStep()
696. took = now() - t
697. if (took > franchiseRun.slowest) { franchiseRun.slowest = took }
698. if (r.error != '') { console.log('franchise seed: ' + r.error) }
699. if (took > 1500) { console.log('franchise seed: slow step, ' + took + ' ms') }
700. if (r.done) {
701. franchiseRun.done = true
702. console.log('franchise seed done: ' + franchiseSeedLine() + ' seconds=' + ((now() - franchiseRun.at) / 1000) + ' slowestStepMs=' + franchiseRun.slowest)
703. return null
704. }
705. franchiseRun.steps = franchiseRun.steps + 1
706. franchiseRun.nextAt = now() + pauseMsAfter(r)
707. if (franchiseRun.steps % franchiseSeedLogEvery == 0) { console.log('franchise seed: ' + franchiseSeedLine() + ' seconds=' + ((now() - franchiseRun.at) / 1000)) }
708. return null
709. }
710. if (!franchiseSeedOn) { console.log('franchise seed: off (TRACKER_FRANCHISE_SEED=0)') }
711. else {
712. if (!syncEnabled) { console.log('franchise seed: no TMDB_READ_TOKEN — builds from the titles we have, imports nothing') }
713. franchiseClock = every(0.1)
714. on franchiseClock.tick(x) {
715. franchiseSeedTick()
716. return null
717. }
718. }