tracker

19.9 KB

1. // people.hl — THE PERSON PAGE's data (tracker.worldapi.org#16, components/person.hl `/person/<slug>`): a person by slug,
2. // their facts and filmography, and the ONE-TIME fetch of their whole filmography from TMDB on the first visit. Creator:
3. // "the movies come from artist scrolls. when one clicks on an artist that has no dataset yet the old version fetched that
4. // artist and its entire bibilogrqphy so its profile was complete".
5. //
6. // personBySlug(slug) the person (slug = `urlSegment`) — a slug → id map built once (15k people, one scan)
7. // personInfoOf(p) name, photo URL, born/died lines, bio
8. // filmographyOf(p) their credits (`p.shows`): poster, title, year, role, kind — newest first, adult titles hidden
9. // personFillStep(id) ONE step of the fetch, called again and again by the page until it answers `done`:
10. // 1st call: GET <TMDB>/person/<tmdbId>?append_to_response=combined_credits (ONE request: details +
11. // every cast/crew credit) + the photo (w185, like the posters); the person's facts are updated;
12. // every next call stores the next `fillBatch` titles; the last one links the credits and marks
13. // the person complete (`filmographyAt`, ms). hl:web serves one request at a time — between two
14. // steps every other request is served, so a person with 300 credits never holds the server.
15. //
16. // COMPLETE = `filmographyAt` set: never fetched again (the daily sync may refresh followed people later — not built).
17. // Migrated people are NOT complete (153 of them carry the old tracker's credits — kept, the fetch adds the rest).
18. // A title we don't have → a MINIMAL show record (title, year, type, tmdbId, `adult`, overview, genres by TMDB id, language;
19. // no seasons, no poster file; `minimal = true` — details.hl completes it when it is opened or by the repair job, tracker#26), shaped like search.hl importTitle's
20. // (`oldId tmdb-<tv|movie>-<id>`, a free slug), indexed for the search and put into the public lists (catalog.hl).
21. // The person's credits (`shows`, the migrated field): [{ show, character, posterPath, adult }] — `character` is the role
22. // text (the characters, then crew jobs like "Director", " / " between them); `posterPath` = TMDB's, so the page can show
23. // TMDB's small poster (w92, like search's web rows) until we have the file. ADULT (TMDB's flag, mission 054): a row is
24. // shown only for a PUBLIC title (shows.hl `isPublicTitle`: `adult == false`, unknown hidden like the public lists) whose
25. // credit is not adult either (mission 056 merge).
26. // ONE PROCESS PER STORAGE (hybriel#21); new rows get self-assigned 16-hex ids (hybriel#113), like tmdbsync.hl.
27. import { now } from 'hl:time'
28. import { fetch } from 'hl:fetch'
29. import { randomBytes } from 'hl:crypto'
30. import { writeFile, mkDir, exists } from 'hl:fs'
31. import { personsTable, showsTable, genresTable, storageDir, showById, showYear, posterUrlOf, isPublicTitle, titlePath, typeWordOf, typeClassOf, liveShowOf } from './shows.hl'
32. import { tmdbGet, tmdbImageBase, textOr } from './tmdbsync.hl'
33. import { titleIdByTmdb, indexShow, slugOf, slugBaseOf, indexPerson } from './search.hl'
34. import { refreshCatalogShow, sortDesc } from './catalog.hl'
35. import { claimShortId } from './shortids.hl'
37. static profilesDir = storageDir + '/profiles'
38. // titles stored per step (real data: see STATUS.md ticket #16 for the per-step time)
39. static fillBatch = 40
41. static listOr = (&v) => { return v == null ? [] : v }
43. // ---- the person -------------------------------------------------------------------------------------------------
44. static slugIndex = { built = false ids = {} }
45. static ensureSlugIndex = () => {
46. if (!slugIndex.built) {
47. let m = {}
48. for (p of personsTable.find(null, null)) { if (p.urlSegment != null) { m[p.urlSegment] = p.id } }
49. slugIndex.ids = m
50. slugIndex.built = true
51. }
52. return true
53. }
54. static personBySlug = (&slug) => {
55. if (slug == null || hlTypeName(slug) != 'String' || slug == '') { return null }
56. ensureSlugIndex()
57. let id = slugIndex.ids[slug]
58. return id == null ? null : personsTable.fetch(id)
59. }
61. static isComplete = (&p) => { return p.filmographyAt != null }
62. // the page asks TMDB for this person: not complete yet, and TMDB knows them
63. static needsFill = (&p) => { return !isComplete(p) && p.tmdbId != null }
65. // the photo: /profiles/<oldId>.<ext> (project.hl profileRoute) — only when the file is there (the migrated `image` = 'jpg'
66. // of 129 people never came with a file)
67. static profileName = (&p) => {
68. if (p.oldId == null || p.image == null || p.image == '') { return null }
69. return p.oldId + '.' + p.image
70. }
71. static photoUrlOf = (&p) => {
72. let name = profileName(p)
73. return name != null && exists(profilesDir + '/' + name) ? '/profiles/' + name : ''
74. }
76. static monthNames = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
77. // '1986-03-04' → '4 March 1986'; a bare year or anything else as it is
78. static dateText = (&d) => {
79. let t = textOr(d)
80. if (t == null) { return null }
81. if (t.length < 10) { return t }
82. let m = toNumber(t.slice(5, 7))
83. let day = toNumber(t.slice(8, 10))
84. if (m == null || day == null || m < 1 || m > 12) { return t }
85. return day + ' ' + monthNames[m - 1] + ' ' + t.slice(0, 4)
86. }
88. // { name, photoUrl, born, died, bio } — every field a plain string ('' = none)
89. static personInfoOf = (&p) => {
90. let born = dateText(p.birthDate)
91. let place = textOr(p.birthPlace)
92. let bornText = ''
93. if (born != null && place != null) { bornText = 'Born ' + born + ' in ' + place } else if (born != null) { bornText = 'Born ' + born } else if (place != null) { bornText = 'Born in ' + place }
94. let died = dateText(p.deathDate)
95. return { name = p.name != null ? p.name : '' photoUrl = photoUrlOf(p) born = bornText died = died != null ? 'Died ' + died : ''
96. bio = textOr(p.biography) != null ? p.biography : '' }
97. }
99. // ---- the filmography --------------------------------------------------------------------------------------------
100. // the credits, one row per title (a title twice — the migrated credit and TMDB's — is ONE row, roles merged), newest
101. // first by the title's release ('YYYY-MM-DD', else its year), undated last; only public titles (no adult/unknown)
102. static filmographyOf = (&p) => {
103. let byShow = {}
104. let order = []
105. for (c of listOr(p.shows)) {
106. if (c != null && c.show != null) {
107. let role = textOr(c.character)
108. let prev = byShow[c.show]
109. if (prev == null) {
110. byShow[c.show] = { role = role != null ? role : '' posterPath = textOr(c.posterPath) adult = c.adult == true }
111. order.push(c.show)
112. } else {
113. let x = prev
114. if (role != null && !(' / ' + x.role + ' / ').includes(' / ' + role + ' / ')) { x.role = x.role == '' ? role : x.role + ' / ' + role }
115. if (x.posterPath == null) { x.posterPath = textOr(c.posterPath) }
116. if (c.adult == true) { x.adult = true }
117. byShow[c.show] = x
118. }
119. }
120. }
121. let list = []
122. let listed = {}
123. for (sid of order) {
124. let c = byShow[sid]
125. // tracker#31: a credit on a merged tombstone is the keeper's (merge.hl repoints them; a keeper listed once)
126. let s = liveShowOf(showById(sid))
127. if (isPublicTitle(s) && !c.adult && listed[s.id] != true) {
128. listed[s.id] = true
129. let y = showYear(s)
130. let release = textOr(s.release)
131. let key = release != null && release.length >= 4 ? release : (y != null ? y : '')
132. let poster = posterUrlOf(s)
133. if (poster == '/posters/none' && c.posterPath != null) { poster = tmdbImageBase + '/w92' + c.posterPath }
134. list.push({ key = key id = s.id title = s.title year = y != null ? y : '' href = titlePath(s) posterUrl = poster
135. role = c.role typeLabel = typeWordOf(s) typeClass = typeClassOf(s) })
136. }
137. }
138. let out = []
139. let undated = []
140. for (r of sortDesc(list)) { if (r.key == '') { undated.push(r) } else { out.push(r) } }
141. for (r of undated) { out.push(r) }
142. return out
143. }
145. // ---- the fetch ------------------------------------------------------------------------------------------------
146. // person id → { items, pos, credits, added, requests, started, steps } while a fetch is under way (in memory: a restart
147. // loses it — the titles stored so far are found again by TMDB id, nothing is duplicated)
148. static jobs = {}
150. // TMDB genre id → ours (our genres carry TMDB's id)
151. static genreMap = { built = false ids = {} }
152. static genreIdsOfTmdb = (&tmdbIds) => {
153. if (!genreMap.built) {
154. let m = {}
155. for (g of genresTable.find(null, null)) { if (g.tmdbId != null) { m['' + g.tmdbId] = g.id } }
156. genreMap.ids = m
157. genreMap.built = true
158. }
159. let out = []
160. for (t of listOr(tmdbIds)) {
161. let id = genreMap.ids['' + t]
162. if (id != null && !out.includes(id)) { out.push(id) }
163. }
164. return out
165. }
167. // TMDB's combined_credits → one item per title (cast AND crew of the same title merged), TV + movies only
168. static creditItems = (&cc) => {
169. let byKey = {}
170. let order = []
171. let all = []
172. if (cc != null) {
173. for (c of listOr(cc.cast)) { all.push({ credit = c role = textOr(c.character) }) }
174. for (c of listOr(cc.crew)) { all.push({ credit = c role = textOr(c.job) }) }
175. }
176. for (a of all) {
177. let c = a.credit
178. let kind = c.media_type
179. if ((kind == 'tv' || kind == 'movie') && c.id != null && hlTypeName(c.id) == 'Number') {
180. let k = kind + ':' + c.id
181. let known = byKey[k]
182. if (known == null) {
183. let name = kind == 'tv' ? textOr(c.name) : textOr(c.title)
184. if (name != null) {
185. byKey[k] = { kind = kind tmdbId = c.id title = name release = textOr(kind == 'tv' ? c.first_air_date : c.release_date)
186. overview = textOr(c.overview) language = textOr(c.original_language) genreIds = listOr(c.genre_ids)
187. posterPath = textOr(c.poster_path) adult = c.adult == true role = a.role != null ? a.role : '' }
188. order.push(k)
189. }
190. } else if (a.role != null && !(' / ' + known.role + ' / ').includes(' / ' + a.role + ' / ')) {
191. let x = known
192. x.role = x.role == '' ? a.role : x.role + ' / ' + a.role
193. byKey[k] = x
194. }
195. }
196. }
197. let out = []
198. for (k of order) { out.push(byKey[k]) }
199. return out
200. }
202. // the photo: w185 → storage/mpackdb/profiles/<oldId>.<ext> — { requests, image, tmdbProfile, failed }
203. static syncProfile = (&p, profilePath) => {
204. let out = { requests = 0 image = p.image tmdbProfile = p.tmdbProfile failed = null }
205. if (textOr(profilePath) == null || p.oldId == null) { return out }
206. let parts = profilePath.split('.')
207. let ext = parts[parts.length - 1].toLowerCase()
208. if (ext != 'jpg' && ext != 'jpeg' && ext != 'png' && ext != 'webp') { return out }
209. let file = profilesDir + '/' + p.oldId + '.' + ext
210. if (exists(file) && p.tmdbProfile == profilePath && p.image == ext) { return out }
211. out.requests = 1
212. let r = fetch(tmdbImageBase + '/w185' + profilePath, { headers = { 'user-agent' = 'tracker.worldapi.org (person page)' } timeoutMs = 20000 })
213. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
214. out.failed = 'photo ' + profilePath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
215. return out
216. }
217. mkDir(profilesDir)
218. writeFile(file, toBytes(r.body), 420)
219. out.image = ext
220. out.tmdbProfile = profilePath
221. return out
222. }
224. // step 1: TMDB's person + credits; the facts (a TMDB value that is empty never overwrites ours) and the photo
225. static startFill = (p) => {
226. let t0 = now()
227. let d = tmdbGet('/person/' + p.tmdbId + '?append_to_response=combined_credits')
228. if (d.failed != null) { return { error = 'TMDB: ' + d.failed } }
229. let items = creditItems(d.combined_credits)
230. let photo = syncProfile(p, d.profile_path)
231. let x = p
232. if (textOr(d.biography) != null) { x.biography = d.biography }
233. if (textOr(d.birthday) != null) { x.birthDate = d.birthday }
234. if (textOr(d.deathday) != null) { x.deathDate = d.deathday }
235. if (textOr(d.place_of_birth) != null) { x.birthPlace = d.place_of_birth }
236. if (textOr(x.imdbId) == null && textOr(d.imdb_id) != null) { x.imdbId = d.imdb_id }
237. if (d.adult != null) { x.adult = d.adult == true }
238. x.image = photo.image
239. x.tmdbProfile = photo.tmdbProfile
240. if (personsTable.update(x.id, x) == null) { return { error = 'could not store the person: ' + personsTable.lastError() } }
241. if (photo.failed != null) { console.log('person fill: ' + p.name + ': ' + photo.failed) }
242. jobs[p.id] = { items = items pos = 0 credits = [] added = 0 requests = 1 + photo.requests started = t0 steps = 1 tmdbMs = now() - t0 }
243. return { done = false pos = 0 total = items.length }
244. }
246. // a title we don't have: the minimal record (shape: search.hl importTitle) → its id, or null
247. static addMinimalTitle = (&cr, &pid, pname) => {
248. let kindType = cr.kind == 'tv' ? 'series' : 'movie'
249. let rel = cr.release
250. let yr = rel != null && rel.length >= 4 ? toNumber(rel.slice(0, 4)) : null
251. let slug = slugOf(cr.title, cr.tmdbId)
252. let record = {
253. id = randomBytes(8, 'hex') title = cr.title summary = null tmdbSummary = cr.overview tmdbId = cr.tmdbId
254. language = cr.language type = kindType release = rel year = yr image = null
255. urlSegment = slug episodesCount = null homepage = null imdbId = null seasonsCount = null
256. status = null tagline = null tvdbId = null tvmzId = null genres = genreIdsOfTmdb(cr.genreIds)
257. cast = [{ person = pid character = cr.role actor = pname }] seasons = [] oldId = 'tmdb-' + cr.kind + '-' + cr.tmdbId
258. adult = cr.adult imported = now() minimal = true
259. }
260. // mission 060: its short id (shortids.hl — unique across shows and persons)
261. record.shortId = claimShortId('show', record.id, record.oldId)
262. if (showsTable.put(record) == null) {
263. console.log('person fill: "' + cr.title + '" not stored: ' + showsTable.lastError())
264. return null
265. }
266. indexShow(record)
267. refreshCatalogShow(record.id)
268. return record.id
269. }
271. // every next step: the next `fillBatch` titles; the last one links the credits and marks the person complete
272. static fillStep = (p, jobIn) => {
273. let job = jobIn
274. let total = job.items.length
275. let end = job.pos + fillBatch
276. if (end > total) { end = total }
277. let credits = job.credits
278. let added = job.added
279. let pid = p.id
280. let pname = p.name
281. let i = job.pos
282. while (i < end) {
283. let item = job.items[i]
284. let sid = titleIdByTmdb(item.kind == 'tv' ? 'series' : 'movie', item.tmdbId)
285. if (sid == null) {
286. sid = addMinimalTitle(item, pid, pname)
287. if (sid != null) { added = added + 1 }
288. }
289. if (sid != null) { credits.push({ show = sid character = item.role posterPath = item.posterPath adult = item.adult }) }
290. i = i + 1
291. }
292. job.pos = end
293. job.credits = credits
294. job.added = added
295. job.steps = job.steps + 1
296. jobs[pid] = job
297. if (end < total) { return { done = false pos = end total = total } }
299. // done: the credits (+ the migrated ones TMDB no longer lists), complete
300. let have = {}
301. let all = []
302. for (c of credits) { have[c.show] = true all.push(c) }
303. for (c of listOr(p.shows)) { if (c != null && c.show != null && have[c.show] != true) { all.push(c) } }
304. let x = p
305. x.shows = all
306. x.filmographyAt = now()
307. if (personsTable.update(x.id, x) == null) { return { error = 'could not store the filmography: ' + personsTable.lastError() } }
308. personsTable.persist()
309. showsTable.persist()
310. jobs[pid] = null
311. console.log('person fill: ' + pname + ' (tmdb ' + p.tmdbId + ') credits=' + total + ' added=' + added + ' requests=' + job.requests + ' steps=' + job.steps + ' tmdbMs=' + job.tmdbMs + ' ms=' + (now() - job.started))
312. return { done = true pos = total total = total added = added }
313. }
315. // one step for the person with this id: { done, pos, total } or { error }
316. static personFillStep = (&personId) => {
317. let p = personsTable.fetch(personId)
318. if (p == null) { return { error = 'no such person' } }
319. if (isComplete(p)) { return { done = true } }
320. if (p.tmdbId == null) { return { error = 'TMDB does not know this person' } }
321. let job = jobs[personId]
322. if (job == null) { return startFill(p) }
323. return fillStep(p, job)
324. }
326. // ---- people from a title's cast (tracker#26, details.hl completeStep) ------------------------------------------
327. // TMDB person id → our person id, built once (15k people, one scan), kept up to date by castPersonId
328. // (flat statics, written key by key like search.hl's haveTmdb: `let m = …ids` would copy the whole map — plain data
329. // deep-copies on assignment)
330. static personTmdbBuilt = { yes = false }
331. static personTmdbIds = {}
332. static personIdByTmdb = (&tmdbId) => {
333. if (!personTmdbBuilt.yes) {
334. for (p of personsTable.find(null, null)) { if (p.tmdbId != null) { personTmdbIds['' + p.tmdbId] = p.id } }
335. personTmdbBuilt.yes = true
336. }
337. return personTmdbIds['' + tmdbId]
338. }
339. // a free person slug: "Maya-Rudolph", taken → "-2", … (the migrated people's form)
340. static personSlugOf = (&name, &tmdbId) => {
341. ensureSlugIndex()
342. let base = slugBaseOf(name, 'person-' + tmdbId)
343. if (slugIndex.ids[base] == null) { return base }
344. let n = 2
345. while (slugIndex.ids[base + '-' + n] != null) { n = n + 1 }
346. return base + '-' + n
347. }
348. // how many people castPersonId added since the start (details.hl counts a title's new people by the difference, tracker#28)
349. static castPeopleAdded = { n = 0 }
350. static castPeopleCount = () => { return castPeopleAdded.n }
351. // our person for one cast entry of TMDB ({ tmdbId, name, character, gender }) — the one we have by TMDB id, else a NEW
352. // person (name, tmdbId, a free slug, this one credit; not complete: their page fetches the rest on the first visit like any
353. // migrated person). Answers the id, or null when it could not be stored. Also a CREW person (tracker#28: `character` = the
354. // job, "Director").
355. static castPersonId = (&c, &showId) => {
356. let have = personIdByTmdb(c.tmdbId)
357. if (have != null) { return have }
358. let record = { id = randomBytes(8, 'hex') name = c.name tmdbId = c.tmdbId urlSegment = personSlugOf(c.name, c.tmdbId) oldId = 'tmdb-person-' + c.tmdbId
359. gender = c.gender image = null tmdbProfile = null adult = false birthDate = null birthPlace = null deathDate = null imdbId = null
360. shows = showId != null ? [{ show = showId character = c.character }] : [] added = now() }
361. // mission 062 (tracker#27 for #26/#28): every person the app creates gets its short id at once (shortids.hl)
362. record.shortId = claimShortId('person', record.id, record.oldId)
363. if (personsTable.put(record) == null) {
364. console.log('details: person "' + c.name + '" not stored: ' + personsTable.lastError())
365. return null
366. }
367. personTmdbIds['' + c.tmdbId] = record.id
368. slugIndex.ids[record.urlSegment] = record.id
369. indexPerson(record)
370. castPeopleAdded.n = castPeopleAdded.n + 1
371. return record.id
372. }
374. // ---- GUEST STARS (mission 062): a series' guests are NOT person records — details.hl stores them on the title ({ tmdbId, actor,
375. // character, profile, episodes, guest }, `person` only when we have them already). 160k people (most of them one-episode guests)
376. // cost 3.3 GB of RSS at boot (w061). A guest's link is `/person/tmdb/<tmdbId>?show=<show id>` (project.hl guestRoute): the FIRST
377. // open creates the person (name + role from that title's guest entry; without one — an old link — TMDB's /person/<id>), with its
378. // short id, then the person page fills it like any other.
379. static guestOf = (&showId, &tmdbId) => {
380. let s = showId != null && hlTypeName(showId) == 'String' && showId != '' ? showsTable.fetch(showId) : null
381. if (s == null) { return null }
382. for (c of listOr(s.cast)) { if (c != null && c.tmdbId == tmdbId && textOr(c.actor) != null) { return c } }
383. return null
384. }
385. // our person for this TMDB person (created now when we have none) → { id, created } or { error }
386. static guestPersonId = (tmdbId, showId) => {
387. let have = personIdByTmdb(tmdbId)
388. if (have != null) { return { id = have created = false } }
389. let g = guestOf(showId, tmdbId)
390. let c = null
391. let inShow = null
392. if (g != null) {
393. c = { tmdbId = tmdbId name = g.actor character = g.character gender = null }
394. inShow = showId
395. } else {
396. let d = tmdbGet('/person/' + tmdbId)
397. if (d.failed != null) { return { error = 'TMDB: ' + d.failed } }
398. if (textOr(d.name) == null || d.adult == true) { return { error = 'no such person' } }
399. c = { tmdbId = tmdbId name = d.name character = null gender = d.gender }
400. }
401. let pid = castPersonId(c, inShow)
402. if (pid == null) { return { error = 'could not store the person' } }
403. personsTable.persist()
404. console.log('guest opened: ' + c.name + ' (tmdb ' + tmdbId + ') → new person ' + pid + (inShow != null ? ' from show ' + inShow : ' from TMDB'))
405. return { id = pid created = true }
406. }