tracker

27.4 KB

1. // project.hl — tracker.worldapi.org: STEP 1 (tracker.worldapi.org#1), an empty shell. Login copied unchanged from
2. // calendar.worldapi.org (rejected once for a centered sign-in and no header selector — architect, 2026-09-27):
3. // ident only, no own passwords (README "How apps use ident" of ident.worldapi.org), the identity selector in the
4. // header (components/main.hl). No shows, no data yet — later steps come from the creator (CONCEPT.md).
5. //
6. // the header's ident-selector / "Log in with ident" -> <ident>/login?key=&return=<this app>/login/callback
7. // /login/callback?ident_code=<code> the app's server exchanges the code for the identity's short id
8. // (POST <ident>/api/exchange) and signs this app's OWN session in
9. // (session.user = { id = <users @id> }, users.hl)
10. // the header's "Log out" button signs this app's session out via the face trackerLogOut (components/main.hl)
11. // — ident's own session, and the browser's ident cookie, are untouched
12. //
13. // Config (env; no committed secret — the first deploy registers this app in ident and sets these, done by the
14. // architect, as with ident's own .env for SMTP):
15. // TRACKER_PORT (45008), TRACKER_URL (this app's own public address, for the return URL),
16. // IDENT_URL (https://ident.worldapi.org), TRACKER_KEY (pk_…), TRACKER_SECRET (sk_…)
17. import WebFramework from 'hl:web'
18. import { Response } from 'hl:http1'
19. import { randomBytes } from 'hl:crypto'
20. import { env } from 'hl:proc'
21. import { readBytes, exists } from 'hl:fs'
22. import { now, timestamp, every } from 'hl:time'
24. import Styles from './styles.hl'
25. import { exchangeCode, ensureUser } from './users.hl'
26. import Home from './components/home.hl'
27. import LoginFailed from './components/loginfailed.hl'
28. import Show from './components/show.hl'
29. import Unwatched from './components/unwatched.hl'
30. import Schedule from './components/schedule.hl'
31. import MyShows from './components/myshows.hl'
32. import MyMovies from './components/mymovies.hl'
33. import Movies from './components/movies.hl'
34. import AllShows from './components/allshows.hl'
35. import Search from './components/search.hl'
36. import Person from './components/person.hl'
37. import Genre from './components/genre.hl'
38. import Franchises from './components/franchises.hl'
39. import Franchise from './components/franchise.hl'
40. import FranchiseEdit from './components/franchiseedit.hl'
41. import TimelineEdit from './components/timelineedit.hl'
42. import { ensureIndex, searchTitleChanged } from './search.hl'
43. import { syncEnabled, syncShow, persistSync, emptyTotals, addTotals, totalsLine, pauseMsAfter, msPerRequest, adultBackfillQueue, backfillTitle } from './tmdbsync.hl'
44. import { allFollowedShowIds } from './follows.hl'
45. import { refreshCatalogShow } from './catalog.hl'
46. import { ensureShortIds, shortIdStep, shortIdPath, shortIdsLeft, shortIdStats, persistShortIds } from './shortids.hl'
47. import { dark, darker } from './shared/tokens.hl'
48. import { ensureFranchises, seedPrepare, seedStep, seedLine, seedPersist } from './franchises.hl'
50. static appTitle = "tracker"
52. // ---- THE INSTALLABLE APP (tracker.worldapi.org#10), the same way calendar.worldapi.org does it: hl:web's own web app
53. // manifest (/__hl/manifest.webmanifest, linked from every head with the apple-touch-icon and theme-color) and service
54. // worker (/__hl/sw.js) from these settings — no JavaScript of ours. Icons in icons/ (icon.svg is the source, the PNGs
55. // are rendered from it with rsvg-convert, README "Icons"). The theme colour is the header's background (darker), the
56. // splash background the page's (dark) — both off the tokens, not written out again.
57. appThemeColor = darker.value
58. appBackgroundColor = dark.value
59. appIcons = [
60. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'any' }
61. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'any' }
62. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'maskable' }
63. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'maskable' }
64. ]
65. appTouchIcon = '/icons/apple-touch-icon.png'
66. appFavicon = '/icons/favicon.svg'
67. // OFFLINE: only the shell — the header and `/` (components/main.hl says "You are offline" while the browser has no
68. // network). The data pages are not kept: without a network they get hl:web's "Unavailable offline" page.
69. offline = [ Home ]
71. styles = Styles
73. port = env('TRACKER_PORT') != null ? toNumber(env('TRACKER_PORT')) : 45008
74. watching = env('TRACKER_WATCH') != '0'
75. // read by hl:web's own manifest config (WebFramework.hl `cfg.sessionDir`), not the constructor call below
76. sessionDir = env('TRACKER_SESSIONS') != null ? env('TRACKER_SESSIONS') : null
78. // ---- THE LOGIN BUTTON'S RETURN (ident README "How apps use ident"), copied from calendar.worldapi.org's project.hl
79. // BACK TO THE PAGE: /login.js puts `?next=` into the button's return URL at the click. Only a same-origin PATH
80. // goes (one `/`, URL-safe characters, ≤ 500). Anything else → `/`.
81. nextChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~/?&=%+,;@!$()*:'
82. safePath = (want) => {
83. if (want == null || hlTypeName(want) != 'String' || want == '' || want.length > 500) { return '/' }
84. if (want.slice(0, 1) != '/' || want.slice(0, 2) == '//' || want.slice(0, 7) == '/login/') { return '/' }
85. let i = 0
86. while (i < want.length) {
87. if (!nextChars.includes(want[i])) { return '/' }
88. i = i + 1
89. }
90. return want
91. }
93. // A FAILED LOGIN is a page (components/loginfailed.hl): the reason is parked in the session, then → /login/failed
94. failed = (req, why) => {
95. let s = req.session
96. let fresh = s == null
97. if (fresh) { s = server.sessions.mint() }
98. s.data.loginError = why
99. server.sessions.save(s)
100. let res = new Response('login failed: ' + why, { status = 302 headers = { 'Location' = '/login/failed' 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
101. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
102. return res
103. }
105. // the function route gets the cookie's session as req.session (hybriel #11); none yet → minted here
106. loginCallback = (route, req) => {
107. if (req.method != 'GET') { return failed(req, 'GET only') }
108. let q = req.query != null ? req.query : {}
109. let code = q.ident_code
110. if (code == null || code == '') { return failed(req, 'ident sent no login code') }
111. let x = exchangeCode(code)
112. if (x.error != null) { return failed(req, x.error) }
113. let u = ensureUser(x.identity)
114. if (u == null) { return failed(req, 'could not store the user') }
115. let s = req.session
116. let fresh = s == null
117. if (fresh) { s = server.sessions.mint() }
118. s.user = { id = u.id }
119. s.data.tag = randomBytes(16)
120. s.data.loginError = null
121. server.sessions.save(s)
122. let res = new Response('logged in', { status = 302 headers = { 'Location' = safePath(q.next) 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
123. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
124. return res
125. }
127. // ---- /posters/<name> (tracker.worldapi.org#4): a show's poster image, served dynamically from
128. // storage/mpackdb/posters/ (the poster's file name is the show's oldId + its migrated extension,
129. // shows.hl posterName) — falls back to a placeholder svg when the real file isn't there yet (the
130. // mongo export never included the actual poster bytes, see the report's `open`).
131. postersDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/posters'
132. placeholderPoster = './assets/poster-placeholder.svg'
133. posterNameChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_'
134. isSafePosterName = (name) => {
135. if (name == null || hlTypeName(name) != 'String' || name == '' || name.length > 100) { return false }
136. let i = 0
137. while (i < name.length) {
138. if (!posterNameChars.includes(name[i])) { return false }
139. i = i + 1
140. }
141. return true
142. }
143. endsWithStr = (s, suffix) => { return s.length >= suffix.length && s.slice(s.length - suffix.length) == suffix }
144. mimeOfPoster = (name) => {
145. if (endsWithStr(name, '.png')) { return 'image/png' }
146. if (endsWithStr(name, '.webp')) { return 'image/webp' }
147. if (endsWithStr(name, '.svg')) { return 'image/svg+xml' }
148. return 'image/jpeg'
149. }
150. posterRoute = (route, req) => {
151. let name = route.params != null ? route.params.name : null
152. if (!isSafePosterName(name)) { return new Response('bad poster name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
153. let path = postersDir + '/' + name
154. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
155. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
156. }
158. // ---- /profiles/<name> (tracker.worldapi.org#16): a person's photo, from storage/mpackdb/profiles/ (people.hl syncProfile
159. // downloads it on the first visit of the person page) — the same checks and placeholder as /posters/<name>
160. profilesDir = (env('TRACKER_STORAGE') != null ? env('TRACKER_STORAGE') : './storage/mpackdb') + '/profiles'
161. profileRoute = (route, req) => {
162. let name = route.params != null ? route.params.name : null
163. if (!isSafePosterName(name)) { return new Response('bad photo name', { status = 400 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
164. let path = profilesDir + '/' + name
165. if (exists(path)) { return new Response(readBytes(path), { headers = { 'Content-Type' = mimeOfPoster(name) 'Cache-Control' = 'public, max-age=86400' } }) }
166. return new Response(readBytes(placeholderPoster), { headers = { 'Content-Type' = 'image/svg+xml' 'Cache-Control' = 'public, max-age=3600' } })
167. }
169. // ---- the old addresses of the personal lists (tracker.worldapi.org#8): they all live under /my/ now —
170. // a permanent redirect so old links keep working (architect's decision)
171. movedTo = (path) => { return new Response('moved to ' + path, { status = 301 headers = { 'Location' = path 'Content-Type' = 'text/plain; charset=utf-8' } }) }
172. unwatchedMoved = (route, req) => { return movedTo('/my/unwatched') }
173. scheduleMoved = (route, req) => { return movedTo('/my/schedule') }
174. // mission 058: the show page's genre pills linked `/genre/<x>` (no page there) — the genre pages are `/genres/<x>`
175. genreMoved = (route, req) => {
176. let g = route.params != null ? route.params.genre : null
177. return movedTo(isSafePosterName(g) ? '/genres/' + g : '/')
178. }
179. // mission 060 (tracker.worldapi.org#27): `/<shortId>` → 301 to the title's or person's page (shortids.hl); upper case is
180. // read as lower case; anything else is a 404
181. shortIdRoute = (route, req) => {
182. let sid = route.params != null ? route.params.sid : null
183. let path = sid != null && hlTypeName(sid) == 'String' ? shortIdPath(sid.toLowerCase()) : null
184. if (path == null) { return new Response('not found', { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
185. return movedTo(path)
186. }
188. routes = [
189. { pattern = "/favicon.ico" file = "./icons/favicon.ico" headers = { 'Cache-Control' = 'no-cache' } }
190. // the installable app (tracker.worldapi.org#10): the icons (manifest and service worker are hl:web's own, from appIcons / offline)
191. { pattern = "/icons/icon-192.png" file = "./icons/icon-192.png" headers = { 'Cache-Control' = 'no-cache' } }
192. { pattern = "/icons/icon-512.png" file = "./icons/icon-512.png" headers = { 'Cache-Control' = 'no-cache' } }
193. { pattern = "/icons/apple-touch-icon.png" file = "./icons/apple-touch-icon.png" headers = { 'Cache-Control' = 'no-cache' } }
194. { pattern = "/icons/favicon.svg" file = "./icons/favicon.svg" headers = { 'Cache-Control' = 'no-cache' } }
195. { pattern = "/login/callback" function = loginCallback }
196. { pattern = "/login/failed" component = LoginFailed }
197. { pattern = "/login.js" file = "./login.js" headers = { 'Cache-Control' = 'no-cache' } }
198. { pattern = "/posters/:name" function = posterRoute }
199. // tracker.worldapi.org#13: every movie / every series, 24 per page — the page is a PATH segment (`/movies/page/2`), a
200. // page component cannot read `?page=` (hybriel#11) — and the movies the user follows
201. { pattern = "/movies" component = Movies }
202. { pattern = "/movies/page/:page" component = Movies }
203. { pattern = "/shows" component = AllShows }
204. { pattern = "/shows/page/:page" component = AllShows }
205. { pattern = "/shows/:slug" component = Show }
206. { pattern = "/my/shows" component = MyShows }
207. { pattern = "/my/movies" component = MyMovies }
208. { pattern = "/my/unwatched" component = Unwatched }
209. { pattern = "/my/schedule" component = Schedule }
210. { pattern = "/unwatched" function = unwatchedMoved }
211. { pattern = "/schedule" function = scheduleMoved }
212. // tracker.worldapi.org#14: the search — `/search/<text>` (hl:web gives a page no query string, so not `?q=`)
213. { pattern = "/search/:q?" component = Search }
214. // tracker.worldapi.org#16: a person's page (the show page's cast and the search link here) and their photo
215. { pattern = "/person/:slug" component = Person }
216. { pattern = "/profiles/:name" function = profileRoute }
217. // mission 058 (tracker.worldapi.org#25): a genre's movies and series, like /movies (the page in the path)
218. { pattern = "/genres/:genre" component = Genre }
219. { pattern = "/genres/:genre/page/:page" component = Genre }
220. { pattern = "/genre/:genre" function = genreMoved }
221. // tracker.worldapi.org#19 (mission 066): franchises and timelines, the creator's editor (`/franchises/edit` before the slug)
222. { pattern = "/franchises" component = Franchises }
223. { pattern = "/franchises/edit" component = FranchiseEdit }
224. { pattern = "/franchises/:slug" component = Franchise }
225. { pattern = "/timelines/:tslug" component = Franchise }
226. { pattern = "/timelines/:slug/edit" component = TimelineEdit }
227. { pattern = "/" component = Home }
228. // mission 060 (tracker.worldapi.org#27): a short id → its page (LAST: every named one-segment route above wins)
229. { pattern = "/:sid" function = shortIdRoute }
230. ]
232. // WHO GETS THE PUSH: the login state reaches the tabs of one session (components/main.hl trackerSignedIn/Out).
233. tagOf = (session) => { return session != null && session.data != null ? session.data.tag : null }
234. audience = {
235. trackerSignedIn = (tag, session) => { return tag != null && tagOf(session) == tag }
236. trackerSignedOut = (tag, session) => { return tag != null && tagOf(session) == tag }
237. }
239. // cookies are per host, not per port: an own name keeps this app's session apart from
240. // ident's own (and from any other worldapi app sharing a dev host), see ident README "Design
241. // tokens" / hybriel#10 hybriel#17.
242. sessionCookie = 'trackersid'
243. // tracker.worldapi.org#14: the search index (search.hl) is built once here at boot (~0.6 s on the real data), not by the
244. // first search
245. searchIndexReady = ensureIndex()
246. // mission 060: every short id in use (one map over shows AND persons) and the records still without one, before the first request
247. shortIdsReady = ensureShortIds()
248. // mission 066: which timelines every title is in (one scan of the timelines), before the first show page
249. franchisesReady = ensureFranchises()
250. server =new WebFramework(routes = routes, styles = styles, port = port, sessionCookie = sessionCookie, watchMode = watching)
252. // ---- THE DAILY TMDB SYNC (tracker.worldapi.org#9, tmdbsync.hl) --------------------------------------
253. // Once a day at TRACKER_SYNC_HOUR (UTC, default 4 — hl:time has no time zones; the container runs UTC) every
254. // followed show is synced from TMDB. hl:web serves ONE request at a time, so the run never takes the server
255. // for itself: a 1-second clock does ONE show per tick (1–3 TMDB requests + maybe its poster) and returns — the
256. // requests that queued meanwhile are served between two ticks, and a tick never starts inside a page render.
257. // After a show that made n requests the next waits n × 260 ms (TMDB: ≤ 40 requests / 10 s) + 550 ms per TVmaze lookup
258. // (≤ 20 / 10 s, tracker.worldapi.org#12).
259. // TRACKER_SYNC=0 switches it off; without TMDB_READ_TOKEN it is off too (logged once).
260. syncHour = env('TRACKER_SYNC_HOUR') != null ? toNumber(env('TRACKER_SYNC_HOUR')) : 4
261. syncOn = env('TRACKER_SYNC') != '0' && syncEnabled
262. syncQueue = []
263. syncAt = 0
264. syncDay = ''
265. syncNextAt = 0
266. syncTotals = emptyTotals()
267. syncTick = () => {
268. let t = now()
269. if (syncQueue.length == 0) {
270. let stamp = timestamp(t)
271. let day = stamp.slice(0, 10)
272. if (toNumber(stamp.slice(11, 13)) != syncHour || day == syncDay) { return null }
273. syncDay = day
274. syncQueue = allFollowedShowIds()
275. syncAt = t
276. syncNextAt = 0
277. syncTotals = emptyTotals()
278. console.log('tmdb sync: start, ' + syncQueue.length + ' followed shows')
279. return null
280. }
281. if (t < syncNextAt) { return null }
282. let id = syncQueue[0]
283. let rest = []
284. let i = 1
285. while (i < syncQueue.length) { rest.push(syncQueue[i]) i = i + 1 }
286. syncQueue = rest
287. let r = syncShow(id)
288. // the public lists (catalog.hl, tracker.worldapi.org#13): this show's place by its newest released episode; the sync
289. // also brings TMDB's adult flag (mission 054) → the search's view of it too
290. refreshCatalogShow(id)
291. searchTitleChanged(id)
292. let took = now() - t
293. syncTotals = addTotals(syncTotals, r)
294. if (r.error != '') { console.log('tmdb sync: ' + r.error) }
295. // a step blocks every request meanwhile (Saturday Night Live, 53 seasons, 4 requests: ~1.7 s) — say so in the log
296. if (took > 1500) { console.log('tmdb sync: slow step, show ' + id + ', ' + r.requests + ' requests, ' + took + ' ms') }
297. syncNextAt = now() + pauseMsAfter(r)
298. if (syncQueue.length == 0 || syncTotals.shows % 10 == 0) { persistSync() }
299. if (syncQueue.length == 0) { console.log('tmdb sync done: ' + totalsLine(syncTotals) + ' seconds=' + ((now() - syncAt) / 1000)) }
300. return null
301. }
303. // ---- THE ADULT-FLAG BACKFILL (mission 054, tmdbsync.hl backfillTitle) -----------------------------------
304. // The public lists and the search show only titles TMDB says are not adult (shows.hl isPublicTitle; unknown = hidden). At
305. // start the app collects every title whose flag is still unknown and asks TMDB for ONE title per step — like the daily
306. // sync, on the same clock, so requests queued meanwhile are served between two steps: one details request (adult,
307. // poster_path, external ids) + the poster file when missing, then a pause of 260 ms per TMDB API request (≤ 40 / 10 s;
308. // the poster comes from TMDB's image host, not the API, and is not counted). Each answered title goes into the lists
309. // (catalog.hl refreshCatalogShow) and the search at once. RESUMES after a restart: the queue is built from what is
310. // still unknown. Stops when the queue is empty; titles TMDB answers 404 for are marked (`adultCheck`) and stay hidden; a
311. // passing failure (no answer, 429, 5xx) goes to the back of the queue, at most 2 more tries per start.
312. // While the daily sync runs, the backfill waits. Log: start, every TRACKER_BACKFILL_LOG_EVERY (500) titles, done.
313. // TRACKER_BACKFILL=0 switches it off; without TMDB_READ_TOKEN it is off too.
314. backfillOn = env('TRACKER_BACKFILL') != '0' && syncEnabled
315. backfillLogEvery = env('TRACKER_BACKFILL_LOG_EVERY') != null ? toNumber(env('TRACKER_BACKFILL_LOG_EVERY')) : 500
316. backfillQueue = null
317. backfillPos = 0
318. backfillDone = false
319. backfillAt = 0
320. backfillNextAt = 0
321. backfillTries = {}
322. backfillTotals = { titles = 0 adult = 0 notAdult = 0 unknown = 0 posters = 0 ids = 0 requests = 0 errors = 0 retries = 0 }
323. backfillLine = () => {
324. let b = backfillTotals
325. return b.titles + '/' + backfillQueue.length + ' titles, adult=' + b.adult + ' notAdult=' + b.notAdult + ' unknown=' + b.unknown + ' posters=' + b.posters
326. + ' ids=' + b.ids + ' requests=' + b.requests + ' errors=' + b.errors + ' retries=' + b.retries + ' seconds=' + ((now() - backfillAt) / 1000)
327. }
328. backfillTick = () => {
329. if (backfillDone) { return null }
330. let t = now()
331. if (backfillQueue == null) {
332. backfillQueue = adultBackfillQueue()
333. backfillAt = t
334. console.log('adult backfill: start, ' + backfillQueue.length + ' titles without an adult flag (' + (now() - t) + ' ms to find them)')
335. if (backfillQueue.length == 0) { backfillDone = true console.log('adult backfill: nothing to do') }
336. return null
337. }
338. if (t < backfillNextAt) { return null }
339. let id = backfillQueue[backfillPos]
340. backfillPos = backfillPos + 1
341. let r = backfillTitle(id)
342. if (r.retry && (backfillTries[id] == null || backfillTries[id] < 2)) {
343. backfillTries[id] = backfillTries[id] == null ? 1 : backfillTries[id] + 1
344. let q = backfillQueue
345. q.push(id)
346. backfillQueue = q
347. backfillTotals.retries = backfillTotals.retries + 1
348. } else {
349. backfillTotals.titles = backfillTotals.titles + 1
350. if (r.adult == true) { backfillTotals.adult = backfillTotals.adult + 1 } else if (r.adult == false) { backfillTotals.notAdult = backfillTotals.notAdult + 1 } else { backfillTotals.unknown = backfillTotals.unknown + 1 }
351. }
352. backfillTotals.posters = backfillTotals.posters + r.posters
353. backfillTotals.ids = backfillTotals.ids + r.ids
354. backfillTotals.requests = backfillTotals.requests + r.requests
355. backfillTotals.errors = backfillTotals.errors + r.errors
356. if (r.error != '') { console.log('adult backfill: ' + r.error) }
357. if (r.adult != null) {
358. refreshCatalogShow(id)
359. searchTitleChanged(id)
360. }
361. let took = now() - t
362. if (took > 1500) { console.log('adult backfill: slow step, show ' + id + ', ' + took + ' ms') }
363. backfillNextAt = now() + r.requests * msPerRequest
364. let finished = backfillPos >= backfillQueue.length
365. if (finished || backfillPos % 25 == 0) { persistSync() }
366. if (!finished && backfillTotals.titles > 0 && backfillTotals.titles % backfillLogEvery == 0 && !r.retry) { console.log('adult backfill: ' + backfillLine()) }
367. if (finished) {
368. backfillDone = true
369. console.log('adult backfill done: ' + backfillLine())
370. }
371. return null
372. }
374. // ONE clock for both (a tick every 0.1 s): the daily sync's step once a second as before; the backfill's steps in between,
375. // only while no daily run is going on
376. clockOn = syncOn || backfillOn
377. clockTicks = 0
378. if (syncOn) { console.log('tmdb sync: daily at ' + syncHour + ':00 UTC') } else { console.log('tmdb sync: off (' + (syncEnabled ? 'TRACKER_SYNC=0' : 'no TMDB_READ_TOKEN') + ')') }
379. if (!backfillOn) { console.log('adult backfill: off (' + (syncEnabled ? 'TRACKER_BACKFILL=0' : 'no TMDB_READ_TOKEN') + ')') }
380. if (clockOn) {
381. syncClock = every(0.1)
382. on syncClock.tick(x) {
383. clockTicks = clockTicks + 1
384. let busy = syncQueue.length > 0
385. if (syncOn && clockTicks % 10 == 0) { syncTick() }
386. if (backfillOn && !busy && syncQueue.length == 0) { backfillTick() }
387. return null
388. }
389. }
391. // ---- THE SHORT-ID BACKFILL (mission 060, tracker.worldapi.org#27, shortids.hl shortIdStep) ----------------------------
392. // Every show and person without a short id gets one in the background: on its own clock (every 0.1 s) ONE batch per tick,
393. // as many records as fit into TRACKER_SHORTID_BUDGET_MS (default 40 ms) — no TMDB, only the tables — so a page waits at
394. // most one batch. The queue is built at start (`ensureShortIds`) from what is still missing = it resumes after a restart.
395. // Persisted every 20 batches and at the end. Log: start, every TRACKER_SHORTID_LOG_EVERY (5000) records, done; with nothing
396. // to do one line. TRACKER_SHORTIDS=0 switches it off (the gate's resume check).
397. shortIdOn = env('TRACKER_SHORTIDS') != '0'
398. shortIdBudget = env('TRACKER_SHORTID_BUDGET_MS') != null ? toNumber(env('TRACKER_SHORTID_BUDGET_MS')) : 40
399. shortIdLogEvery = env('TRACKER_SHORTID_LOG_EVERY') != null ? toNumber(env('TRACKER_SHORTID_LOG_EVERY')) : 5000
400. shortIdRun = { at = 0 batches = 0 assigned = 0 nextLog = 0 slowest = 0 slowestPersist = 0 done = false }
401. shortIdTick = () => {
402. if (shortIdRun.done) { return null }
403. let t = now()
404. let r = shortIdStep(shortIdBudget)
405. let took = now() - t
406. shortIdRun.batches = shortIdRun.batches + 1
407. shortIdRun.assigned = shortIdRun.assigned + r.assigned
408. if (took > shortIdRun.slowest) { shortIdRun.slowest = took }
409. if (r.done || shortIdRun.batches % 20 == 0) {
410. let tp = now()
411. persistShortIds()
412. if (now() - tp > shortIdRun.slowestPersist) { shortIdRun.slowestPersist = now() - tp }
413. }
414. if (!r.done && shortIdRun.assigned >= shortIdRun.nextLog) {
415. console.log('short ids: ' + shortIdRun.assigned + ' assigned, ' + r.left + ' left, ' + ((now() - shortIdRun.at) / 1000) + ' s')
416. shortIdRun.nextLog = shortIdRun.nextLog + shortIdLogEvery
417. }
418. if (r.done) {
419. shortIdRun.done = true
420. let s = shortIdStats()
421. console.log('short ids done: assigned=' + shortIdRun.assigned + ' oldKept=' + s.oldKept + ' batches=' + shortIdRun.batches + ' slowestBatchMs=' + shortIdRun.slowest + ' slowestPersistMs=' + shortIdRun.slowestPersist + ' seconds=' + ((now() - shortIdRun.at) / 1000))
422. }
423. return null
424. }
425. if (!shortIdOn) { console.log('short ids: backfill off (TRACKER_SHORTIDS=0), ' + shortIdsLeft() + ' records without one') }
426. else if (shortIdsLeft() == 0) { console.log('short ids: nothing to do') }
427. else {
428. shortIdRun.at = now()
429. shortIdRun.nextLog = shortIdLogEvery
430. console.log('short ids: backfill start, ' + shortIdsLeft() + ' records, ' + shortIdBudget + ' ms per batch')
431. shortIdClock = every(0.1)
432. on shortIdClock.tick(x) {
433. shortIdTick()
434. return null
435. }
436. }
438. // ---- THE COLLECTION SEED (mission 066, tracker.worldapi.org#19, franchises.hl seedStep) ------------------------------------
439. // TMDB's movie collections become timelines: ONE movie per step (its details; once per collection the collection's parts), on
440. // its own 0.1 s clock like the adult backfill — a pause of 260 ms per TMDB request made (≤ 40 / 10 s). It WAITS while the daily
441. // sync or the adult backfill runs (they share TMDB's limit and the backfill brings the adult flags it skips by). The queue =
442. // every non-adult movie without a check row, so it RESUMES after a restart and does nothing once all are checked. Persisted
443. // every TRACKER_COLLECTIONS_PERSIST_EVERY (25) steps and at the end. Log: start, every TRACKER_COLLECTIONS_LOG_EVERY (500) movies, done.
444. // TRACKER_COLLECTIONS=0 switches it off; without TMDB_READ_TOKEN it is off too.
445. collectionsOn = env('TRACKER_COLLECTIONS') != '0' && syncEnabled
446. collectionsLogEvery = env('TRACKER_COLLECTIONS_LOG_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_LOG_EVERY')) : 500
447. collectionsPersistEvery = env('TRACKER_COLLECTIONS_PERSIST_EVERY') != null ? toNumber(env('TRACKER_COLLECTIONS_PERSIST_EVERY')) : 25
448. collectionRun = { at = 0 steps = 0 nextAt = 0 done = false started = false slowest = 0 }
449. collectionTick = () => {
450. if (collectionRun.done) { return null }
451. if (syncQueue.length > 0 || (backfillOn && !backfillDone)) { return null }
452. let t = now()
453. if (!collectionRun.started) {
454. collectionRun.started = true
455. collectionRun.at = t
456. let n = seedPrepare()
457. console.log('collections: start, ' + n + ' movies to check (' + (now() - t) + ' ms to find them)')
458. if (n == 0) { collectionRun.done = true console.log('collections: nothing to do') }
459. return null
460. }
461. if (t < collectionRun.nextAt) { return null }
462. let r = seedStep()
463. if (r.done) {
464. collectionRun.done = true
465. seedPersist()
466. console.log('collections done: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000) + ' slowestStepMs=' + collectionRun.slowest)
467. return null
468. }
469. collectionRun.steps = collectionRun.steps + 1
470. let took = now() - t
471. if (took > collectionRun.slowest) { collectionRun.slowest = took }
472. if (r.error != '') { console.log('collections: ' + r.error) }
473. if (took > 1500) { console.log('collections: slow step, ' + took + ' ms') }
474. collectionRun.nextAt = now() + r.requests * msPerRequest
475. if (collectionRun.steps % collectionsPersistEvery == 0) { seedPersist() }
476. if (collectionRun.steps % collectionsLogEvery == 0) { console.log('collections: ' + seedLine() + ' seconds=' + ((now() - collectionRun.at) / 1000)) }
477. return null
478. }
479. if (!collectionsOn) { console.log('collections: off (' + (syncEnabled ? 'TRACKER_COLLECTIONS=0' : 'no TMDB_READ_TOKEN') + ')') }
480. else {
481. collectionClock = every(0.1)
482. on collectionClock.tick(x) {
483. collectionTick()
484. return null
485. }
486. }
488. on Error(e) { console.log('error absorbed: ' + e.message) }