tracker

46.6 KB

1. // lib/jobs.hl — THE BACKGROUND JOBS (mission 028: out of project.hl, unchanged): the daily TMDB sync run, the adult and kind
2. // backfills, the details repair, the credits job, the duplicate merge, the short-id backfill and the collection seed — each with
3. // its queue, its pause, its log lines, and the clocks that drive them (which job runs when, which one waits for which) — and, last,
4. // the curated franchise seed (tracker#40). The work of ONE step is the topic's own (sync.hl, backfill.hl, details.hl, credits.hl,
5. // merge.hl, shortids.hl, collections.hl, franchiseseed.hl).
6. // A CLASS, not statics: its state (queues, positions, totals) is rewritten on every step, and a static cannot be reassigned —
7. // project.hl makes the one instance (`jobs = new Jobs()`) after the server, as these lines ran there before; constructing it
8. // starts the clocks and logs which jobs are off.
9. import { env } from 'hl:proc'
10. import { readFile, exists } from 'hl:fs'
11. import { now, timestamp, every } from 'hl:time'
12. import { syncEnabled, pauseMsAfter, msPerRequest } from './tmdb.hl'
13. import { persistSync } from './sync.hl'
14. import { emptyTotals, addTotals, totalsLine } from './sync-helpers.hl'
15. import { planStart, planStep, planIds, planLine, markSynced, syncRunStep } from './dailysync.hl'
16. import { adultBackfillQueue, backfillTitle, kindBackfillQueue, kindTitle, tvmazeStatusQueue, tvmazeStatusTitle, artQueue, artTitle } from './backfill.hl'
17. import { photoQueue, photoStep, persistPhotos } from './people.hl'
18. import { detailsRepairQueue, completeStep, persistDetails, applySynced } from './details.hl'
19. import { persistCredits, creditsQueue, creditsStep } from './credits.hl'
20. import { allFollowedShowIds } from './follows.hl'
21. import { refreshCatalogShow } from './catalog.hl'
22. import { searchTitleChanged, searchTitleRenamed } from './search.hl'
23. import { mergePrepare, mergeStep, mergeStats, mergeLine, persistMerge } from './merge.hl'
24. import { shortIdStep, shortIdStats, persistShortIds, shortIdsLeft } from './shortids.hl'
25. import { seedPrepare, seedStep, seedLine, seedPersist } from './collections.hl'
26. import { franchiseSeedPrepare, franchiseSeedStep, franchiseSeedLine, franchiseSeedStats } from './franchiseseed.hl'
28. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, sync.hl; tracker#18, dailysync.hl) -------------------------
29. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC). hl:web serves ONE
30. // request at a time, so the run never takes the server for itself: a 1-second clock does ONE step per tick and returns — the
31. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
32. // tracker#18: the run first asks TMDB's and TVmaze's CHANGE LISTS since the last run (dailysync.hl — one list page per step)
33. // and then syncs only the titles of ours that changed, followed ones first — every title stays current, not only the followed
34. // ones. FULL WALK (every followed show, as before #18) on the first run, after a gap > 14 days, or when a change list fails.
35. // A title's step: sync.hl syncShow (1–3 TMDB requests + maybe its poster, TVmaze) + details.hl applySynced (the record's
36. // title, overview, tagline … and cast/crew from the same details answer). After a step that made n requests the next waits
37. // n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze request (≤ 20 / 10 s, tracker.worldapi.org#12).
38. // The finished run stores its day (dailysync.hl markSynced). Log: "tmdb sync: start, …", the change lists ("tmdb sync:
39. // changes since …"), errors, slow steps, "tmdb sync done: mode=… changeRequests=… <totals>".
40. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
41. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
42. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
43. syncQueue = []
44. syncPlanning = false
45. syncMode = ''
46. syncMark = false
47. syncChangeRequests = 0
48. syncAt = 0
49. syncDay = ''
50. syncNextAt = 0
51. syncTotals = emptyTotals()
52. syncFinish = () => {
53. persistDetails()
54. if (syncMark) { markSynced(syncDay) }
55. console.log('tmdb sync done: mode=' + syncMode + ' changeRequests=' + syncChangeRequests + ' ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000))
56. return null
57. }
58. syncTick = () => {
59. t = now()
60. if (syncQueue.length == 0 && !syncPlanning) {
61. stamp = timestamp(t)
62. day = stamp.slice(0, 10)
63. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
64. syncDay = day
65. syncAt = t
66. syncNextAt = 0
67. syncTotals = emptyTotals()
68. syncChangeRequests = 0
69. p = planStart(day, null)
70. syncMode = p.mode
71. syncMark = true
72. if (p.mode == 'delta') {
73. syncPlanning = true
74. console.log('tmdb sync: start, changes ' + p.why)
75. return null
76. }
77. syncQueue = allFollowedShowIds()
78. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows, full walk (' + p.why + ')')
79. if (syncQueue.length == 0) { syncFinish() }
80. return null
81. }
82. if (t < syncNextAt) { return null }
83. if (syncPlanning) {
84. ps = planStep()
85. syncChangeRequests = syncChangeRequests + ps.requests + ps.tvmazeRequests
86. syncTotals.requests = syncTotals.requests + ps.requests
87. syncTotals.tvmazeRequests = syncTotals.tvmazeRequests + ps.tvmazeRequests
88. syncNextAt = now() + pauseMsAfter(ps)
89. if (!ps.done) { return null }
90. syncPlanning = false
91. console.log('tmdb sync: ' + planLine())
92. if (ps.failed != '') {
93. // a change list could not be read: the old full walk, and the day is not advanced (the next run asks again)
94. syncMode = 'full'
95. syncMark = false
96. syncQueue = allFollowedShowIds()
97. console.log('tmdb sync: change list failed, full walk of ' + syncQueue.length + ' followed shows instead')
98. } else {
99. syncQueue = planIds()
100. }
101. if (syncQueue.length == 0) { syncFinish() }
102. return null
103. }
104. id = syncQueue[0]
105. rest = []
106. let i = 1
107. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
108. syncQueue = rest
109. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze); tracker#35: one only on
110. // TVmaze's change list its TVmaze status (dailysync.hl syncRunStep)
111. r = syncRunStep(id, syncMode == 'delta')
112. // tracker#18 + #28: the record's own fields (title, overview, tagline …) and the cast + crew from the same details answer
113. if (r.details != null) {
114. c = applySynced(id, r.details)
115. if (c.error != '') { console.log('tmdb sync: ' + c.error) }
116. if (c.renamed) { searchTitleRenamed(id) }
117. }
118. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
119. // also brings TMDB's adult flag (mission 054) → the search's view of it too
120. refreshCatalogShow(id)
121. searchTitleChanged(id)
122. took = now() - t
123. syncTotals = addTotals(syncTotals, r)
124. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
125. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
126. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
127. syncNextAt = now() + pauseMsAfter(r)
128. if (syncQueue.length > 0 && syncTotals.shows % 10 == 0) { persistDetails() }
129. if (syncQueue.length == 0) { syncFinish() }
130. return null
131. }
133. // ---- THE ADULT-FLAG BACKFILL (mission 054, backfill.hl backfillTitle) -----------------------------------
134. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
135. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
136. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
137. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
138. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
139. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
140. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
141. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
142. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
143. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
144. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
145. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
146. backfillQueue = null
147. backfillPos = 0
148. backfillDone = false
149. backfillAt = 0
150. backfillNextAt = 0
151. backfillTries = {}
152. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
153. backfillLine = () => {
154. b = backfillTotals
155. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
156. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
157. }
158. backfillTick = () => {
159. if (backfillDone) { return null }
160. t = now()
161. if (backfillQueue == null) {
162. backfillQueue = adultBackfillQueue()
163. backfillAt = t
164. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
165. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
166. return null
167. }
168. if (t < backfillNextAt) { return null }
169. id = backfillQueue[backfillPos]
170. backfillPos = backfillPos + 1
171. r = backfillTitle(id)
172. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
173. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
174. q = backfillQueue
175. q.push(id)
176. backfillQueue = q
177. backfillTotals.retries = backfillTotals.retries + 1
178. } else {
179. backfillTotals.titles = backfillTotals.titles + 1
180. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
181. }
182. backfillTotals.posters = backfillTotals.posters + r.posters
183. backfillTotals.ids = backfillTotals.ids + r.ids
184. backfillTotals.requests = backfillTotals.requests + r.requests
185. backfillTotals.errors = backfillTotals.errors + r.errors
186. if (r.error != '') { console.log('adult backfill: ' + r.error) }
187. if (r.adult != null) {
188. refreshCatalogShow(id)
189. searchTitleChanged(id)
190. }
191. took = now() - t
192. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
193. backfillNextAt = now() + r.requests * msPerRequest
194. finished = backfillPos >= backfillQueue.length
195. if (finished || backfillPos % 25 == 0) { persistSync() }
196. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
197. if (finished) {
198. backfillDone = true
199. console.log('adult backfill done: ' + backfillLine())
200. }
201. return null
202. }
204. // ---- THE DETAILS REPAIR (tracker.worldapi.org#26, details.hl completeStep) ----------------------------------------
205. // Titles the person pages added from a filmography (#16) are MINIMAL (no poster, seasons, ids, cast …), and (tracker#33, mission
206. // 029) the old tracker's migrated rows were never synced (5,697 on the live copy, 691 series without seasons) — shows.hl
207. // isIncomplete. Opening one completes it (components/show.hl); this job completes the rest in the background: at start it
208. // collects every incomplete, non-adult title (the visibly reduced ones first, details.hl detailsRepairQueue) and does ONE per
209. // step on the app's clock, like the adult backfill — a step is the daily sync's work for one title
210. // (1–3 TMDB requests + poster + TVmaze), requests queued meanwhile are served between two steps; then a pause of 260 ms per
211. // TMDB request + 550 ms per TVmaze request (pauseMsAfter: ≤ 40 / 10 s, ≤ 20 / 10 s). RESUMES after a restart (the queue is
212. // what is still incomplete); TMDB 404 marks the title (`detailsCheck`), a passing failure goes to the back of the queue (at
213. // most 2 more tries per start). Waits while the daily sync or the adult backfill runs. Persisted every 10 titles + at the end.
214. // Log: start, every TRACKER_REPAIR_LOG_EVERY (100) titles, each failure, done. TRACKER_REPAIR=0 switches it off (off too
215. // without TMDB_READ_TOKEN). TRACKER_REPAIR_FAST=1 (the gate, fake TMDB only): 20 ms pause instead.
216. repairOn = env('TRACKER_REPAIR') != '0' && syncEnabled
217. repairFast = env('TRACKER_REPAIR_FAST') == '1'
218. repairLogEvery = env('TRACKER_REPAIR_LOG_EVERY') != null ? toNumber(env('TRACKER_REPAIR_LOG_EVERY')) : 100
219. repairQueue = null
220. repairPos = 0
221. repairDone = false
222. repairAt = 0
223. repairNextAt = 0
224. repairTries = {}
225. repairTotals = { titles = 0 complete = 0 failed = 0 adult = 0 newSeasons = 0 newEpisodes = 0 posters = 0 cast = 0 requests = 0 tvmazeRequests = 0 retries = 0 }
226. repairLine = () => {
227. b = repairTotals
228. return b.titles + '/' + repairQueue.length + ' titles, complete=' + b.complete + ' failed=' + b.failed + ' adult=' + b.adult + ' newSeasons=' + b.newSeasons + ' newEpisodes=' + b.newEpisodes
229. + ' posters=' + b.posters + ' cast=' + b.cast + ' requests=' + b.requests + ' tvmazeRequests=' + b.tvmazeRequests + ' retries=' + b.retries + ' seconds=' + ((now() - repairAt) / 1000)
230. }
231. repairTick = () => {
232. if (repairDone) { return null }
233. t = now()
234. if (repairQueue == null) {
235. repairQueue = detailsRepairQueue()
236. repairAt = t
237. console.log('details repair: start, ' + repairQueue.length + ' incomplete titles (' + (now() - t) + ' ms to find them)')
238. if (repairQueue.length == 0) { repairDone = true console.log('details repair: nothing to do') }
239. return null
240. }
241. if (t < repairNextAt) { return null }
242. id = repairQueue[repairPos]
243. r = completeStep(id)
244. took = now() - t
245. if (took > 1500) { console.log('details repair: slow step, show ' + id + ', ' + took + ' ms') }
246. if (r.error != '') { console.log('details repair: ' + r.error) }
247. repairNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
248. // a series takes several steps: the same title again next time
249. if (!r.done && !r.retry) { return null }
250. repairPos = repairPos + 1
251. if (r.retry && (repairTries[id] == null || repairTries[id] < 2)) {
252. repairTries[id] = repairTries[id] == null ? 1 : repairTries[id] + 1
253. q = repairQueue
254. q.push(id)
255. repairQueue = q
256. repairTotals.retries = repairTotals.retries + 1
257. } else if (!r.skipped) {
258. // done: its counts (a failed first step: the request alone)
259. sum = r.sum
260. repairTotals.titles = repairTotals.titles + 1
261. if (r.adult == true) { repairTotals.adult = repairTotals.adult + 1 } else if (sum.steps == 0) { repairTotals.failed = repairTotals.failed + 1 } else { repairTotals.complete = repairTotals.complete + 1 }
262. repairTotals.newSeasons = repairTotals.newSeasons + sum.newSeasons
263. repairTotals.newEpisodes = repairTotals.newEpisodes + sum.newEpisodes
264. repairTotals.posters = repairTotals.posters + sum.posters
265. repairTotals.cast = repairTotals.cast + sum.cast
266. repairTotals.requests = repairTotals.requests + (sum.steps == 0 ? r.requests : sum.requests)
267. repairTotals.tvmazeRequests = repairTotals.tvmazeRequests + sum.tvmazeRequests
268. }
269. finished = repairPos >= repairQueue.length
270. if (finished || repairPos % 10 == 0) { persistDetails() }
271. if (!finished && !r.skipped && !r.retry && repairTotals.titles > 0 && repairTotals.titles % repairLogEvery == 0) { console.log('details repair: ' + repairLine()) }
272. if (finished) {
273. repairDone = true
274. console.log('details repair done: ' + repairLine())
275. }
276. return null
277. }
279. // ---- THE TVMAZE STATUS BACKFILL (tracker#35, mission 030, backfill.hl tvmazeStatusTitle) — the details repair's second part ----
280. // The page's status pill shows TVmaze's status when a series has one (fresher than TMDB's). The sync stores it whenever it merges
281. // TVmaze's episodes (followed series, every completion); the series it never reached get it here: once the details repair is
282. // done (same switch, TRACKER_REPAIR) every series with a TVmaze id and no tvmazeStatus, ONE TVmaze request per step, then 550 ms
283. // (TVmaze: ≤ 20 / 10 s). RESUMES after a restart (the queue = what still has none); a 404 marks the series
284. // (`tvmazeStatusCheck`), a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily
285. // sync runs. Persisted every 25 series + at the end. Log: start, every TRACKER_REPAIR_LOG_EVERY series, done.
286. statusQueue = null
287. statusPos = 0
288. statusDone = false
289. statusAt = 0
290. statusNextAt = 0
291. statusTries = {}
292. statusTotals = { series = 0 stored = 0 notFound = 0 requests = 0 errors = 0 retries = 0 }
293. statusLine = () => {
294. b = statusTotals
295. return b.series + '/' + statusQueue.length + ' series, stored=' + b.stored + ' notFound=' + b.notFound + ' tvmazeRequests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - statusAt) / 1000)
296. }
297. statusTick = () => {
298. if (statusDone) { return null }
299. t = now()
300. if (statusQueue == null) {
301. statusQueue = tvmazeStatusQueue()
302. statusAt = t
303. console.log('tvmaze status: start, ' + statusQueue.length + ' series without a TVmaze status (' + (now() - t) + ' ms to find them)')
304. if (statusQueue.length == 0) { statusDone = true console.log('tvmaze status: nothing to do') }
305. return null
306. }
307. if (t < statusNextAt) { return null }
308. id = statusQueue[statusPos]
309. statusPos = statusPos + 1
310. r = tvmazeStatusTitle(id)
311. if (r.retry && (statusTries[id] == null || statusTries[id] < 2)) {
312. statusTries[id] = statusTries[id] == null ? 1 : statusTries[id] + 1
313. q = statusQueue
314. q.push(id)
315. statusQueue = q
316. statusTotals.retries = statusTotals.retries + 1
317. } else {
318. statusTotals.series = statusTotals.series + 1
319. if (r.status != null) { statusTotals.stored = statusTotals.stored + 1 } else if (r.errors > 0) { statusTotals.notFound = statusTotals.notFound + 1 }
320. }
321. statusTotals.requests = statusTotals.requests + r.tvmazeRequests
322. statusTotals.errors = statusTotals.errors + r.errors
323. if (r.error != '') { console.log('tvmaze status: ' + r.error) }
324. statusNextAt = now() + (repairFast ? 20 : pauseMsAfter(r))
325. finished = statusPos >= statusQueue.length
326. if (finished || statusPos % 25 == 0) { persistSync() }
327. if (!finished && !r.retry && statusTotals.series > 0 && statusTotals.series % repairLogEvery == 0) { console.log('tvmaze status: ' + statusLine()) }
328. if (finished) {
329. statusDone = true
330. console.log('tvmaze status done: ' + statusLine())
331. }
332. return null
333. }
335. // ---- THE KIND BACKFILL (tracker.worldapi.org#21, mission 068, backfill.hl kindTitle) ----------------------------------
336. // Every TV title without a stored kind gets one: a title with a show genre (Reality / Talk / News) from the stored data (no
337. // request), any other from TMDB's TV type (ONE request: GET /tv/<id>, then 260 ms pause). Starts once the adult backfill is
338. // done (or off), waits while the daily sync runs, ONE title per step on the same clock. A title that becomes a SHOW moves from
339. // /series to /shows at once (catalog.hl refreshCatalogShow). RESUMES after a restart (the queue = every TV title still without
340. // a kind); a 404 stores the genres' answer + `kindCheck`; a passing failure goes to the back of the queue, at most 2 more tries
341. // per start. Log: start, every TRACKER_KINDS_LOG_EVERY (500) titles, done. TRACKER_KINDS=0 switches it off; without
342. // TMDB_READ_TOKEN it is off too (the lists still split by genre: shows.hl titleKind).
343. kindsOn = env('TRACKER_KINDS') != '0' && syncEnabled
344. kindsLogEvery = env('TRACKER_KINDS_LOG_EVERY') != null ? toNumber(env('TRACKER_KINDS_LOG_EVERY')) : 500
345. kindQueue = null
346. kindPos = 0
347. kindDone = false
348. kindAt = 0
349. kindNextAt = 0
350. kindTries = {}
351. kindTotals = { titles = 0 series = 0 shows = 0 fromGenre = 0 moved = 0 requests = 0 errors = 0 retries = 0 }
352. kindLine = () => {
353. k = kindTotals
354. return k.titles + '/' + kindQueue.length + ' titles, series=' + k.series + ' shows=' + k.shows + ' fromGenre=' + k.fromGenre + ' moved=' + k.moved
355. + ' requests=' + k.requests + ' errors=' + k.errors + ' retries=' + k.retries + ' seconds=' + ((now() - kindAt) / 1000)
356. }
357. kindTick = () => {
358. if (kindDone) { return null }
359. t = now()
360. if (kindQueue == null) {
361. kindQueue = kindBackfillQueue()
362. kindAt = t
363. console.log('kind backfill: start, ' + kindQueue.length + ' TV titles without a kind (' + (now() - t) + ' ms to find them)')
364. if (kindQueue.length == 0) { kindDone = true console.log('kind backfill: nothing to do') }
365. return null
366. }
367. if (t < kindNextAt) { return null }
368. id = kindQueue[kindPos]
369. kindPos = kindPos + 1
370. r = kindTitle(id)
371. if (r.retry && (kindTries[id] == null || kindTries[id] < 2)) {
372. kindTries[id] = kindTries[id] == null ? 1 : kindTries[id] + 1
373. q = kindQueue
374. q.push(id)
375. kindQueue = q
376. kindTotals.retries = kindTotals.retries + 1
377. } else {
378. kindTotals.titles = kindTotals.titles + 1
379. if (r.kind == 'show') { kindTotals.shows = kindTotals.shows + 1 } else if (r.kind == 'series') { kindTotals.series = kindTotals.series + 1 }
380. if (r.fromGenre) { kindTotals.fromGenre = kindTotals.fromGenre + 1 }
381. }
382. kindTotals.requests = kindTotals.requests + r.requests
383. kindTotals.errors = kindTotals.errors + r.errors
384. if (r.error != '') { console.log('kind backfill: ' + r.error) }
385. // until now the genres decided (no show genre = series): only a title TMDB's type makes a SHOW changes its list
386. if (r.kind == 'show' && !r.fromGenre) {
387. refreshCatalogShow(id)
388. kindTotals.moved = kindTotals.moved + 1
389. }
390. took = now() - t
391. if (took > 1500) { console.log('kind backfill: slow step, show ' + id + ', ' + took + ' ms') }
392. kindNextAt = now() + r.requests * msPerRequest
393. finished = kindPos >= kindQueue.length
394. if (finished || kindPos % 25 == 0) { persistSync() }
395. if (!finished && kindTotals.titles > 0 && kindTotals.titles % kindsLogEvery == 0 && !r.retry) { console.log('kind backfill: ' + kindLine()) }
396. if (finished) {
397. kindDone = true
398. console.log('kind backfill done: ' + kindLine())
399. }
400. return null
401. }
403. // ---- THE CREDITS JOB (tracker.worldapi.org#28, credits.hl creditsStep) ------------------------------------------------------
404. // Every PUBLIC title (adult == false; adult/unknown are skipped) with a TMDB id gets its whole cast + crew once: at start (after
405. // the adult backfill and the details repair are done) the job collects every such title without `creditsAt` and asks ONE per
406. // step on the app's clock — one request (details + credits / aggregate_credits), then 260 ms (≤ 40 / 10 s); requests queued
407. // meanwhile are served between two steps. Titles the details repair, the daily sync or an import already did carry `creditsAt`
408. // and are not asked. RESUMES after a restart (the queue = what still has no credits); TMDB 404 / adult → `creditsCheck`, never
409. // asked again; a passing failure goes to the back of the queue (at most 2 more tries per start). Waits while the daily sync
410. // runs. Persisted (shows + people only) every 25 titles + at the end. Log: start, every TRACKER_CREDITS_LOG_EVERY (500) titles, failures, slow steps
411. // (> 1.5 s), done. TRACKER_CREDITS=0 switches it off (off too without TMDB_READ_TOKEN). TRACKER_CREDITS_FAST=1 (the gate, fake
412. // TMDB only): 20 ms pause instead.
413. // MEMORY LIMIT: the runtime never frees what a fetch + JSON parse took (mission 061 measured ~6 MB more RSS per title on the real
414. // data, 300 fetches of a 440 KB JSON = +2.7 GB in a 10-line script) — the whole job in one process would add ~35 GB. Before every
415. // step the job reads its own RSS (/proc/self/status); above TRACKER_CREDITS_MAX_MB (16000) it PAUSES for this process (logged) and
416. // goes on with what is left after the next restart (deploy, `docker restart`).
417. creditsOn = env('TRACKER_CREDITS') != '0' && syncEnabled
418. creditsFast = env('TRACKER_CREDITS_FAST') == '1'
419. creditsLogEvery = env('TRACKER_CREDITS_LOG_EVERY') != null ? toNumber(env('TRACKER_CREDITS_LOG_EVERY')) : 500
420. creditsQueueIds = null
421. creditsPos = 0
422. creditsDone = false
423. creditsAt = 0
424. creditsNextAt = 0
425. creditsTries = {}
426. creditsMaxMb = env('TRACKER_CREDITS_MAX_MB') != null ? toNumber(env('TRACKER_CREDITS_MAX_MB')) : 16000
427. creditsTotals = { titles = 0 done = 0 failed = 0 cast = 0 crew = 0 people = 0 requests = 0 retries = 0 slowest = 0 }
428. // this process' resident memory in MB (VmRSS), 0 when unknown
429. rssMb = () => {
430. if (!exists('/proc/self/status')) { return 0 }
431. t = readFile('/proc/self/status')
432. i = t != null ? t.indexOf('VmRSS:') : -1
433. if (i < 0) { return 0 }
434. kb = toNumber(t.slice(i + 6, t.indexOf('kB', i)).trim())
435. return kb != null ? (kb - kb % 1024) / 1024 : 0
436. }
437. creditsLine = () => {
438. b = creditsTotals
439. return b.titles + '/' + creditsQueueIds.length + ' titles, done=' + b.done + ' failed=' + b.failed + ' cast=' + b.cast + ' crew=' + b.crew + ' people=' + b.people
440. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - creditsAt) / 1000)
441. }
442. creditsTick = () => {
443. if (creditsDone) { return null }
444. t = now()
445. if (creditsQueueIds == null) {
446. creditsQueueIds = creditsQueue()
447. creditsAt = t
448. console.log('credits job: start, ' + creditsQueueIds.length + ' titles without credits (' + (now() - t) + ' ms to find them)')
449. if (creditsQueueIds.length == 0) { creditsDone = true console.log('credits job: nothing to do') }
450. return null
451. }
452. if (t < creditsNextAt) { return null }
453. mb = rssMb()
454. if (mb > creditsMaxMb) {
455. creditsDone = true
456. persistCredits()
457. console.log('credits job: paused, RSS ' + mb + ' MB > TRACKER_CREDITS_MAX_MB ' + creditsMaxMb + ' — ' + (creditsQueueIds.length - creditsPos) + ' titles left for the next start; ' + creditsLine())
458. return null
459. }
460. id = creditsQueueIds[creditsPos]
461. creditsPos = creditsPos + 1
462. r = creditsStep(id)
463. took = now() - t
464. if (took > creditsTotals.slowest) { creditsTotals.slowest = took }
465. if (took > 1500) { console.log('credits job: slow step, show ' + id + ', ' + took + ' ms, cast=' + r.cast + ' crew=' + r.crew + ' people=' + r.people) }
466. if (r.error != '') { console.log('credits job: ' + r.error) }
467. creditsNextAt = now() + (creditsFast ? 20 : r.requests * msPerRequest)
468. if (r.retry && (creditsTries[id] == null || creditsTries[id] < 2)) {
469. creditsTries[id] = creditsTries[id] == null ? 1 : creditsTries[id] + 1
470. q = creditsQueueIds
471. q.push(id)
472. creditsQueueIds = q
473. creditsTotals.retries = creditsTotals.retries + 1
474. } else if (!r.skipped) {
475. creditsTotals.titles = creditsTotals.titles + 1
476. if (r.error != '') { creditsTotals.failed = creditsTotals.failed + 1 } else { creditsTotals.done = creditsTotals.done + 1 }
477. creditsTotals.cast = creditsTotals.cast + r.cast
478. creditsTotals.crew = creditsTotals.crew + r.crew
479. creditsTotals.people = creditsTotals.people + r.people
480. }
481. creditsTotals.requests = creditsTotals.requests + r.requests
482. finished = creditsPos >= creditsQueueIds.length
483. if (finished || creditsPos % 25 == 0) { persistCredits() }
484. if (!finished && !r.skipped && !r.retry && creditsTotals.titles > 0 && creditsTotals.titles % creditsLogEvery == 0) { console.log('credits job: ' + creditsLine()) }
485. if (finished) {
486. creditsDone = true
487. console.log('credits job done: ' + creditsLine())
488. }
489. return null
490. }
492. // ---- THE DUPLICATE MERGE (tracker.worldapi.org#31, mission 026, merge.hl) ----------------------------------------------------
493. // The old tracker's duplicate titles (same type + TMDB id) become ONE record each — follows, watches, seasons, cast, credits and
494. // timelines moved to the keeper, the others kept as tombstones whose slug and short id lead to it — and seasons that name a title
495. // without being in its list are merged into their twin or linked, and (tracker#39) a season's double episodes into one (merge.hl
496. // has the rules). On its own clock (every 0.1 s) ONE
497. // item per tick (a duplicate group, or one title's stray seasons; no TMDB), so pages are served between two items. The TMDB jobs
498. // below (daily sync, backfills, repair, credits, collections) wait until it is done: none of them works on a record that is about
499. // to become a tombstone. RESUMES after a restart (the queue = what is still duplicated / stray). Persisted every 10 items and at
500. // the end. Log: start, one line per tombstone / repaired title, done; with nothing to do one line. TRACKER_MERGE=0 switches it off.
501. mergeOn = env('TRACKER_MERGE') != '0'
502. mergeRun = { started = false done = !mergeOn at = 0 items = 0 slowest = 0 }
503. mergeTick = () => {
504. if (mergeRun.done) { return null }
505. t = now()
506. if (!mergeRun.started) {
507. mergeRun.started = true
508. mergeRun.at = t
509. n = mergePrepare()
510. st = mergeStats()
511. console.log('merge: start, ' + st.groups + ' duplicate groups (' + st.records + ' records), ' + st.strayTitles + ' titles with stray seasons, ' + st.doubleSeasons + ' seasons with double episodes (' + st.ms + ' ms to find them)')
512. if (n == 0) {
513. mergeRun.done = true
514. console.log('merge: nothing to do')
515. }
516. return null
517. }
518. r = mergeStep()
519. mergeRun.items = mergeRun.items + 1
520. took = now() - t
521. if (took > mergeRun.slowest) { mergeRun.slowest = took }
522. if (took > 1500) { console.log('merge: slow step, ' + took + ' ms') }
523. if (r.done || mergeRun.items % 10 == 0) { persistMerge() }
524. if (r.done) {
525. mergeRun.done = true
526. console.log('merge done: ' + mergeLine() + ' items=' + mergeRun.items + ' slowestStepMs=' + mergeRun.slowest + ' seconds=' + ((now() - mergeRun.at) / 1000))
527. }
528. return null
529. }
530. if (!mergeOn) { console.log('merge: off (TRACKER_MERGE=0)') }
531. else {
532. mergeClock = every(0.1)
533. on mergeClock.tick(x) {
534. mergeTick()
535. return null
536. }
537. }
539. // ONE clock for all five (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in
540. // between, only while no daily run is going on; the details repair only when neither has work (then its TVmaze status part,
541. // tracker#35); then the kind backfill (#21);
542. // the credits job (#28) last
543. clockOn = syncOn || backfillOn || repairOn || kindsOn || creditsOn
544. clockTicks = 0
545. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
546. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
547. if (!repairOn) { console.log('details repair: off (' + (syncEnabled ? 'TRACKER_REPAIR=0' : 'no TMDB_READ_TOKEN') + ')') }
548. if (!creditsOn) { console.log('credits job: off (' + (syncEnabled ? 'TRACKER_CREDITS=0' : 'no TMDB_READ_TOKEN') + ')') }
549. if (!kindsOn) { console.log('kind backfill: off (' + (syncEnabled ? 'TRACKER_KINDS=0' : 'no TMDB_READ_TOKEN') + ')') }
550. if (clockOn) {
551. syncClock = every(0.1)
552. on syncClock.tick(x) {
553. clockTicks = clockTicks + 1
554. // tracker#31: not before the duplicate merge is done
555. if (!mergeRun.done) { return null }
556. busy = syncQueue.length > 0 || syncPlanning
557. if (syncOn && clockTicks % 10 == 0) { syncTick() }
558. free = !busy && syncQueue.length == 0 && !syncPlanning
559. if (backfillOn && free && !backfillDone) { backfillTick() }
560. else if (repairOn && free && (!backfillOn || backfillDone) && !repairDone) { repairTick() }
561. else if (repairOn && free && (!backfillOn || backfillDone) && !statusDone) { statusTick() }
562. else if (kindsOn && free && (!backfillOn || backfillDone) && (!repairOn || (repairDone && statusDone)) && !kindDone) { kindTick() }
563. else if (creditsOn && free && (!backfillOn || backfillDone) && (!repairOn || (repairDone && statusDone)) && (!kindsOn || kindDone)) { creditsTick() }
564. return null
565. }
566. }
568. // ---- THE SHORT-ID BACKFILL (mission 060, tracker.worldapi.org#27, shortids.hl shortIdStep) ----------------------------
569. // Every show and person without a short id gets one in the background: on its own clock (every 0.1 s) ONE batch per tick,
570. // as many records as fit into TRACKER_SHORTID_BUDGET_MS (default 40 ms) — no TMDB, only the tables — so a page waits at
571. // most one batch. The queue is built at start (`ensureShortIds`) from what is still missing = it resumes after a restart.
572. // Persisted every 20 batches and at the end. Log: start, every TRACKER_SHORTID_LOG_EVERY (5000) records, done; with nothing
573. // to do one line. TRACKER_SHORTIDS=0 switches it off (the gate's resume check).
574. shortIdOn = env('TRACKER_SHORTIDS') != '0'
575. shortIdBudget = env('TRACKER_SHORTID_BUDGET_MS') != null ? toNumber(env('TRACKER_SHORTID_BUDGET_MS')) : 40
576. shortIdLogEvery = env('TRACKER_SHORTID_LOG_EVERY') != null ? toNumber(env('TRACKER_SHORTID_LOG_EVERY')) : 5000
577. shortIdRun = { at = 0 batches = 0 assigned = 0 nextLog = 0 slowest = 0 slowestPersist = 0 done = false }
578. shortIdTick = () => {
579. if (shortIdRun.done) { return null }
580. t = now()
581. r = shortIdStep(shortIdBudget)
582. took = now() - t
583. shortIdRun.batches = shortIdRun.batches + 1
584. shortIdRun.assigned = shortIdRun.assigned + r.assigned
585. if (took > shortIdRun.slowest) { shortIdRun.slowest = took }
586. if (r.done || shortIdRun.batches % 20 == 0) {
587. tp = now()
588. persistShortIds()
589. if (now() - tp > shortIdRun.slowestPersist) { shortIdRun.slowestPersist = now() - tp }
590. }
591. if (!r.done && shortIdRun.assigned >= shortIdRun.nextLog) {
592. console.log('short ids: ' + shortIdRun.assigned + ' assigned, ' + r.left + ' left, ' + ((now() - shortIdRun.at) / 1000) + ' s')
593. shortIdRun.nextLog = shortIdRun.nextLog + shortIdLogEvery
594. }
595. if (r.done) {
596. shortIdRun.done = true
597. s = shortIdStats()
598. console.log('short ids done: assigned=' + shortIdRun.assigned + ' oldKept=' + s.oldKept + ' batches=' + shortIdRun.batches + ' slowestBatchMs=' + shortIdRun.slowest + ' slowestPersistMs=' + shortIdRun.slowestPersist + ' seconds=' + ((now() - shortIdRun.at) / 1000))
599. }
600. return null
601. }
602. if (!shortIdOn) { console.log('short ids: backfill off (TRACKER_SHORTIDS=0), ' + shortIdsLeft() + ' records without one') }
603. else if (shortIdsLeft() == 0) { console.log('short ids: nothing to do') }
604. else {
605. shortIdRun.at = now()
606. shortIdRun.nextLog = shortIdLogEvery
607. console.log('short ids: backfill start, ' + shortIdsLeft() + ' records, ' + shortIdBudget + ' ms per batch')
608. shortIdClock = every(0.1)
609. on shortIdClock.tick(x) {
610. shortIdTick()
611. return null
612. }
613. }
615. // ---- THE COLLECTION SEED (mission 066, tracker.worldapi.org#19, collections.hl seedStep) ------------------------------------
616. // TMDB's movie collections become timelines: ONE movie per step (its details; once per collection the collection's parts), on
617. // its own 0.1 s clock like the adult backfill — a pause of 260 ms per TMDB request made (≤ 40 / 10 s). It WAITS while the daily
618. // sync or the adult backfill runs (they share TMDB's limit and the backfill brings the adult flags it skips by); since the merge
619. // (mission 070) also until the details repair, the kind backfill and the credits job are done (or paused) — the LAST job. The queue =
620. // every non-adult movie without a check row, so it RESUMES after a restart and does nothing once all are checked. Persisted
621. // every TRACKER_COLLECTIONS_PERSIST_EVERY (25) steps and at the end. Log: start, every TRACKER_COLLECTIONS_LOG_EVERY (500) movies, done.
622. // TRACKER_COLLECTIONS=0 switches it off; without TMDB_READ_TOKEN it is off too.
623. collectionsOn = env('TRACKER_COLLECTIONS') != '0' && syncEnabled
624. collectionsLogEvery = env('TRACKER_COLLECTIONS_LOG_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_LOG_EVERY')) : 500
625. collectionsPersistEvery = env('TRACKER_COLLECTIONS_PERSIST_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_PERSIST_EVERY')) : 25
626. collectionRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
627. collectionTick = () => {
628. if (collectionRun.done) { return null }
629. // mission 070 (merge): also behind the details repair, the kind backfill and the credits job — one TMDB pacing at a time
630. if (!mergeRun.done || syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && (!repairDone || !statusDone)) || (kindsOn && !kindDone) || (creditsOn && !creditsDone)) { return null }
631. t = now()
632. if (!collectionRun.started) {
633. collectionRun.started = true
634. collectionRun.at = t
635. n = seedPrepare()
636. console.log('collections: start, ' + n + ' movies to check (' + (now() - t) + ' ms to find them)')
637. if (n == 0) { collectionRun.done = true console.log('collections: nothing to do') }
638. return null
639. }
640. if (t < collectionRun.nextAt) { return null }
641. r = seedStep()
642. if (r.done) {
643. collectionRun.done = true
644. seedPersist()
645. console.log('collections done: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000) + ' slowestStepMs=' + collectionRun.slowest)
646. return null
647. }
648. collectionRun.steps = collectionRun.steps + 1
649. took = now() - t
650. if (took > collectionRun.slowest) { collectionRun.slowest = took }
651. if (r.error != '') { console.log('collections: ' + r.error) }
652. if (took > 1500) { console.log('collections: slow step, ' + took + ' ms') }
653. collectionRun.nextAt = now() + r.requests * msPerRequest
654. if (collectionRun.steps % collectionsPersistEvery == 0) { seedPersist() }
655. if (collectionRun.steps % collectionsLogEvery == 0) { console.log('collections: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000)) }
656. return null
657. }
658. if (!collectionsOn) { console.log('collections: off (' + (syncEnabled ? 'TRACKER_COLLECTIONS=0' : 'no TMDB_READ_TOKEN') + ')') }
659. else {
660. collectionClock = every(0.1)
661. on collectionClock.tick(x) {
662. collectionTick()
663. return null
664. }
665. }
667. // ---- THE CURATED FRANCHISE SEED (tracker.worldapi.org#40, mission 034, franchiseseed.hl) --------------------------------------
668. // data/franchises.json → franchises + timelines, movies AND series. The LAST start job: it waits until the merge, the daily sync,
669. // the backfills, the details repair, the kind backfill, the credits job and the collection seed are done (or off) — the
670. // collection seed's timelines are what the file's `collections` attach. Then on its own 0.1 s clock ONE step per tick: the next
671. // missing title imported (the search's import, complete with cast + crew; its log line `search import: …`), then the pause the daily sync makes after the same
672. // requests (≤ 40 TMDB / 10 s, ≤ 20 TVmaze / 10 s); the queue done, the build (no TMDB). RESUMES after a restart (the queue = the
673. // titles still missing), IDEMPOTENT (nothing missing + nothing changed = no write). Without TMDB_READ_TOKEN it only builds from
674. // what we have. Log: start, every import, every TRACKER_FRANCHISE_SEED_LOG_EVERY (25) imports, done.
675. // TRACKER_FRANCHISE_SEED=0 switches it off; TRACKER_FRANCHISE_FILE another data file (the gate's).
676. franchiseSeedOn = env('TRACKER_FRANCHISE_SEED') != '0'
677. franchiseSeedLogEvery = env('TRACKER_FRANCHISE_SEED_LOG_EVERY') != null ? toNumber(env('TRACKER_FRANCHISE_SEED_LOG_EVERY')) : 25
678. franchiseRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
679. franchiseSeedTick = () => {
680. if (franchiseRun.done) { return null }
681. if (!mergeRun.done || syncQueue.length > 0 || syncPlanning || (backfillOn && !backfillDone) || (repairOn && (!repairDone || !statusDone)) || (kindsOn && !kindDone) || (creditsOn && !creditsDone) || (collectionsOn && !collectionRun.done)) { return null }
682. t = now()
683. if (!franchiseRun.started) {
684. franchiseRun.started = true
685. franchiseRun.at = t
686. n = franchiseSeedPrepare(syncEnabled)
687. st = franchiseSeedStats()
688. if (!st.file) {
689. franchiseRun.done = true
690. console.log('franchise seed: ' + st.error + ' — nothing seeded')
691. return null
692. }
693. console.log('franchise seed: start, ' + st.franchises + ' franchises, ' + st.timelines + ' timelines, ' + st.titles + ' titles in the file — ' + st.have + ' we have, ' + n + ' to import, ' + st.missing + ' TMDB lacks (' + (now() - t) + ' ms)')
694. return null
695. }
696. if (t < franchiseRun.nextAt) { return null }
697. r = franchiseSeedStep()
698. took = now() - t
699. if (took > franchiseRun.slowest) { franchiseRun.slowest = took }
700. if (r.error != '') { console.log('franchise seed: ' + r.error) }
701. if (took > 1500) { console.log('franchise seed: slow step, ' + took + ' ms') }
702. if (r.done) {
703. franchiseRun.done = true
704. console.log('franchise seed done: ' + franchiseSeedLine() + ' seconds=' + ((now() - franchiseRun.at) / 1000) + ' slowestStepMs=' + franchiseRun.slowest)
705. return null
706. }
707. franchiseRun.steps = franchiseRun.steps + 1
708. franchiseRun.nextAt = now() + pauseMsAfter(r)
709. if (franchiseRun.steps % franchiseSeedLogEvery == 0) { console.log('franchise seed: ' + franchiseSeedLine() + ' seconds=' + ((now() - franchiseRun.at) / 1000)) }
710. return null
711. }
712. if (!franchiseSeedOn) { console.log('franchise seed: off (TRACKER_FRANCHISE_SEED=0)') }
713. else {
714. if (!syncEnabled) { console.log('franchise seed: no TMDB_READ_TOKEN — builds from the titles we have, imports nothing') }
715. franchiseClock = every(0.1)
716. on franchiseClock.tick(x) {
717. franchiseSeedTick()
718. return null
719. }
720. }
722. // ---- THE ART BACKFILL + THE PHOTO BACKFILL (mission 032: tracker#33's The Remaining, tracker#37's /people) --------------------
723. // The LAST start jobs, after the franchise seed (and everything it waits for): first every public title without a poster file /
724. // public movie without runtime (backfill.hl artTitle: the poster, else the backdrop, + runtime), then every person without a photo
725. // source (people.hl photoStep, /people's order). ONE item per step on their own 0.1 s clock, then n × 260 ms (≤ 40 TMDB requests
726. // / 10 s). Resume after a restart (`artAt`/`artCheck`, `photoCheck`); a passing failure goes to the back of the queue (at most 2
727. // more tries per start). Persisted every 25 items and at the end; the photo job pauses above TRACKER_PHOTOS_MAX_MB (16000) RSS
728. // like the credits job. Log: start, every TRACKER_ART_LOG_EVERY (100) / TRACKER_PHOTOS_LOG_EVERY (1000) items, done.
729. // TRACKER_ART=0 / TRACKER_PHOTOS=0 switch them off; without TMDB_READ_TOKEN they are off.
730. artOn = env('TRACKER_ART') != '0' && syncEnabled
731. photosOn = env('TRACKER_PHOTOS') != '0' && syncEnabled
732. artLogEvery = env('TRACKER_ART_LOG_EVERY') != null ? toNumber(env('TRACKER_ART_LOG_EVERY')) : 100
733. photosLogEvery = env('TRACKER_PHOTOS_LOG_EVERY') != null ? toNumber(env('TRACKER_PHOTOS_LOG_EVERY')) : 1000
734. photosMaxMb = env('TRACKER_PHOTOS_MAX_MB') != null ? toNumber(env('TRACKER_PHOTOS_MAX_MB')) : 16000
735. artRun = { queue = null pos = 0 done = !artOn at = 0 nextAt = 0 tries = {} titles = 0 posters = 0 backdrops = 0 runtimes = 0 failed = 0 requests = 0 retries = 0 slowest = 0 }
736. photoRun = { queue = null pos = 0 done = !photosOn at = 0 nextAt = 0 tries = {} people = 0 found = 0 none = 0 failed = 0 requests = 0 retries = 0 slowest = 0 }
737. artLine = () => {
738. b = artRun
739. return b.titles + '/' + b.queue.length + ' titles, posters=' + b.posters + ' backdrops=' + b.backdrops + ' runtimes=' + b.runtimes + ' failed=' + b.failed
740. + ' requests=' + b.requests + ' retries=' + b.retries + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - b.at) / 1000)
741. }
742. photoLine = () => {
743. b = photoRun
744. return b.people + '/' + b.queue.length + ' people, photos=' + b.found + ' none=' + b.none + ' failed=' + b.failed + ' requests=' + b.requests + ' retries=' + b.retries
745. + ' slowestStepMs=' + b.slowest + ' seconds=' + ((now() - b.at) / 1000)
746. }
747. // everything the two wait for: the other start jobs (one TMDB pacing at a time)
748. lastJobsMayRun = () => {
749. return mergeRun.done && syncQueue.length == 0 && !syncPlanning && (!backfillOn || backfillDone) && (!repairOn || (repairDone && statusDone)) && (!kindsOn || kindDone)
750. && (!creditsOn || creditsDone) && (!collectionsOn || collectionRun.done) && (!franchiseSeedOn || franchiseRun.done)
751. }
752. artTick = () => {
753. t = now()
754. if (artRun.queue == null) {
755. artRun.queue = artQueue()
756. artRun.at = t
757. console.log('art backfill: start, ' + artRun.queue.length + ' public titles without a poster file or a movie runtime (' + (now() - t) + ' ms to find them)')
758. if (artRun.queue.length == 0) { artRun.done = true console.log('art backfill: nothing to do') }
759. return null
760. }
761. if (t < artRun.nextAt) { return null }
762. id = artRun.queue[artRun.pos]
763. artRun.pos = artRun.pos + 1
764. r = artTitle(id)
765. took = now() - t
766. if (took > artRun.slowest) { artRun.slowest = took }
767. if (r.error != '') { console.log('art backfill: ' + r.error) }
768. artRun.nextAt = now() + r.requests * msPerRequest
769. artRun.requests = artRun.requests + r.requests
770. if (r.retry && (artRun.tries[id] == null || artRun.tries[id] < 2)) {
771. artRun.tries[id] = artRun.tries[id] == null ? 1 : artRun.tries[id] + 1
772. q = artRun.queue
773. q.push(id)
774. artRun.queue = q
775. artRun.retries = artRun.retries + 1
776. } else if (r.requests > 0) {
777. artRun.titles = artRun.titles + 1
778. artRun.posters = artRun.posters + r.posters
779. artRun.backdrops = artRun.backdrops + r.backdrops
780. if (r.runtime != null) { artRun.runtimes = artRun.runtimes + 1 }
781. if (r.error != '') { artRun.failed = artRun.failed + 1 }
782. if (r.posters + r.backdrops > 0) { refreshCatalogShow(id) }
783. }
784. finished = artRun.pos >= artRun.queue.length
785. if (finished || artRun.pos % 25 == 0) { persistSync() }
786. if (!finished && artRun.titles > 0 && artRun.pos % artLogEvery == 0) { console.log('art backfill: ' + artLine()) }
787. if (finished) {
788. artRun.done = true
789. console.log('art backfill done: ' + artLine())
790. }
791. return null
792. }
793. photoTick = () => {
794. t = now()
795. if (photoRun.queue == null) {
796. photoRun.queue = photoQueue()
797. photoRun.at = t
798. console.log('photo backfill: start, ' + photoRun.queue.length + ' people without a photo source (' + (now() - t) + ' ms to find them)')
799. if (photoRun.queue.length == 0) { photoRun.done = true console.log('photo backfill: nothing to do') }
800. return null
801. }
802. if (t < photoRun.nextAt) { return null }
803. mb = rssMb()
804. if (mb > photosMaxMb) {
805. photoRun.done = true
806. persistPhotos()
807. console.log('photo backfill: paused, RSS ' + mb + ' MB > TRACKER_PHOTOS_MAX_MB ' + photosMaxMb + ' — ' + (photoRun.queue.length - photoRun.pos) + ' people left for the next start; ' + photoLine())
808. return null
809. }
810. id = photoRun.queue[photoRun.pos]
811. photoRun.pos = photoRun.pos + 1
812. r = photoStep(id)
813. took = now() - t
814. if (took > photoRun.slowest) { photoRun.slowest = took }
815. if (r.error != '') { console.log('photo backfill: ' + r.error) }
816. photoRun.nextAt = now() + r.requests * msPerRequest
817. photoRun.requests = photoRun.requests + r.requests
818. if (r.retry && (photoRun.tries[id] == null || photoRun.tries[id] < 2)) {
819. photoRun.tries[id] = photoRun.tries[id] == null ? 1 : photoRun.tries[id] + 1
820. q = photoRun.queue
821. q.push(id)
822. photoRun.queue = q
823. photoRun.retries = photoRun.retries + 1
824. } else if (r.requests > 0) {
825. photoRun.people = photoRun.people + 1
826. if (r.found) { photoRun.found = photoRun.found + 1 } else if (r.error != '') { photoRun.failed = photoRun.failed + 1 } else { photoRun.none = photoRun.none + 1 }
827. }
828. finished = photoRun.pos >= photoRun.queue.length
829. if (finished || photoRun.pos % 25 == 0) { persistPhotos() }
830. if (!finished && photoRun.people > 0 && photoRun.pos % photosLogEvery == 0) { console.log('photo backfill: ' + photoLine()) }
831. if (finished) {
832. photoRun.done = true
833. console.log('photo backfill done: ' + photoLine())
834. }
835. return null
836. }
837. if (!artOn) { console.log('art backfill: off (' + (syncEnabled ? 'TRACKER_ART=0' : 'no TMDB_READ_TOKEN') + ')') }
838. if (!photosOn) { console.log('photo backfill: off (' + (syncEnabled ? 'TRACKER_PHOTOS=0' : 'no TMDB_READ_TOKEN') + ')') }
839. if (artOn || photosOn) {
840. lastClock = every(0.1)
841. on lastClock.tick(x) {
842. if (!lastJobsMayRun()) { return null }
843. if (!artRun.done) { artTick() } else if (!photoRun.done) { photoTick() }
844. return null
845. }
846. }