tracker

15.6 KB

1. // lib/details.hl — A TITLE'S FULL DETAILS (tracker.worldapi.org#26). Creator: "when i go to an artist and it loads the bibliography
2. // those loaded movies and series miss most information and it stays that way when i click on them".
3. //
4. // The person page's fill (people.hl, #16) adds every title we don't have as a MINIMAL record — title, year, overview, genres,
5. // type, tmdbId, adult, this one person as cast; no poster, no seasons, no external ids, no homepage/tagline/status
6. // (`minimal = true`). A search import (search.hl importTitle) is full except for the cast. Here a minimal title gets what a
7. // full import has, plus its cast:
8. //
9. // GET <TMDB>/movie/<tmdbId>?append_to_response=external_ids,credits a movie
10. // GET <TMDB>/tv/<tmdbId>?append_to_response=external_ids,aggregate_credits a series (cast of ALL seasons, roles merged)
11. // then sync.hl `syncShowPart(id, <that answer>, …)` — the daily sync's work, without asking for the details again, IN
12. // STEPS (see `jobs` below): a series' seasons + episodes (`season/1,…`), then the poster (w342), missing imdb/tvdb ids,
13. // TVmaze lookup + its episode merge (#15); a movie its imdb id + poster
14. //
15. // Stored (`withDetailsFields`, shared with the daily sync since tracker#18): title, overview → `tmdbSummary` (`summary` is the
16. // creator's own text, never written), homepage, tagline, status, genres (by TMDB id, when
17. // TMDB gives any), release/year/language when we have none, `adult`, the CAST (TMDB's first `castLimit`, in TMDB's order;
18. // a person we have by TMDB id is linked, a new one is added — people.hl castPersonId; cast entries we had and TMDB does not
19. // list are kept after them). Then `detailsAt` (ms) = COMPLETE, never asked again (the daily sync keeps a FOLLOWED title fresh).
20. // TMDB 404 → `detailsCheck = 'TMDB 404'` (never asked again; the page shows what it has). TMDB says ADULT → the flag is
21. // stored, `detailsCheck = 'adult'`, no poster/seasons (mission 056: an adult title is never public, its poster not kept).
22. //
23. // tracker#33 (mission 029): INCOMPLETE is every title TMDB's details never went through (shows.hl isIncomplete) — also the old
24. // tracker's migrated rows (5,697 on the live copy: never synced, 691 series without seasons); they are completed the same way.
25. //
26. // WHO CALLS IT (`completeStep`, again and again until `done`): the show page when an incomplete title is opened
27. // (components/show.hl face `showComplete`, skeleton meanwhile) and the repair job in jobs.hl (`repairTick`, one step per tick
28. // on the app's clock, like the adult backfill) for every incomplete title that exists when the app starts. Adult titles are
29. // left alone by both.
30. //
31. // The cast and crew of the same answer: credits.hl (withCredits). Mission 028: the cast + crew parts moved there, `isIncomplete`
32. // to shows.hl.
33. import { now } from 'hl:time'
34. import { env } from 'hl:proc'
35. import { textOr, listOr } from './util.hl'
36. import { showsTable, personsTable, kindOfTitle, isIncomplete, missingParts } from './shows.hl'
37. import { tmdbGet, adultOf } from './tmdb.hl'
38. import { syncShowWith, syncShowPart, persistSync } from './sync.hl'
39. import { searchTitleChanged, importTitle } from './search.hl'
40. import { refreshCatalogShow } from './catalog.hl'
41. import { genreIdsOfTmdb } from './people.hl'
42. import { withCredits, applyCredits } from './credits.hl'
43. import { hasCredits } from './credits-helpers.hl'
45. // what the show page and the repair complete: an incomplete title that is not adult (mission 054/056: never public)
46. static needsDetails = (&s) => { return isIncomplete(s) && s.adult != true && s.mergedInto == null }
48. // every title the repair job has to do (read once when it starts — so a restart goes on with what is left). tracker#33: the ones
49. // a page shows as reduced first (shows.hl missingParts: no seasons, poster, genres …), then the rest, each in table order
50. static detailsRepairQueue = () => {
51. first = []
52. rest = []
53. for (s of showsTable.find(null, null)) {
54. if (needsDetails(s)) {
55. if (missingParts(s).length > 0) { first.push(s.id) } else { rest.push(s.id) }
56. }
57. }
58. for (id of rest) { first.push(id) }
59. return first
60. }
62. // THE RECORD'S OWN FIELDS from a details answer (tracker#26 completion; tracker#18 the daily sync too — creator: "some shows changed
63. // descriptions and titles and that did not update"): title (a series' `name`, a movie's `title`), TMDB's overview → `tmdbSummary`,
64. // homepage, tagline, status, genres (by TMDB id), adult; type / release / year / language only when we have none. A TMDB value
65. // that is empty never overwrites ours. `summary` is the creator's OWN text — never written here (the page shows summary >
66. // tmdbSummary > tvmazeSummary, shows.hl summaryOf). The slug (`urlSegment`) never changes with the title.
67. // mission 070 (conductor): the sync never DESTROYS what the old tracker's migration brought — before TMDB's value replaces a
68. // migrated record's title / genres / homepage / tagline for the FIRST time, the old one is kept in `migratedTitle` /
69. // `migratedGenres` / `migratedHomepage` / `migratedTagline` (kept, shown nowhere; set once, a later rename leaves it). A record is
70. // migrated when its `oldId` is the old tracker's (not `tmdb-…`: search import, filmography fill). `status`, counts, ids and the
71. // poster are TMDB's state, written by the sync since #9 — not kept.
72. static isMigratedRecord = (&show) => { return show.oldId != null && ('' + show.oldId).slice(0, 5) != 'tmdb-' }
73. static sameIdList = (&a, &b) => {
74. x = listOr(a)
75. y = listOr(b)
76. if (x.length != y.length) { return false }
77. let i = 0
78. while (i < x.length) { if ('' + x[i] != '' + y[i]) { return false } i = i + 1 }
79. return true
80. }
81. static withDetailsFields = (show, &d, &movie) => {
82. x = show
83. migrated = isMigratedRecord(show)
84. adult = adultOf(d)
85. if (adult != null) { x.adult = adult }
86. let title = textOr(movie ? d.title : d.name)
87. if (title != null) {
88. if (migrated && x.migratedTitle == null && textOr(x.title) != null && x.title != title) { x.migratedTitle = x.title }
89. x.title = title
90. }
91. if (textOr(d.overview) != null) { x.tmdbSummary = d.overview }
92. if (textOr(d.homepage) != null) {
93. if (migrated && x.migratedHomepage == null && textOr(x.homepage) != null && x.homepage != d.homepage) { x.migratedHomepage = x.homepage }
94. x.homepage = d.homepage
95. }
96. if (textOr(d.tagline) != null) {
97. if (migrated && x.migratedTagline == null && textOr(x.tagline) != null && x.tagline != d.tagline) { x.migratedTagline = x.tagline }
98. x.tagline = d.tagline
99. }
100. if (textOr(d.status) != null) { x.status = d.status }
101. tmdbGenreIds = []
102. for (g of listOr(d.genres)) { if (g != null && g.id != null) { tmdbGenreIds.push(g.id) } }
103. genres = genreIdsOfTmdb(tmdbGenreIds)
104. if (genres.length > 0) {
105. if (migrated && x.migratedGenres == null && listOr(x.genres).length > 0 && !sameIdList(x.genres, genres)) { x.migratedGenres = x.genres }
106. x.genres = genres
107. }
108. if (textOr(x.type) == null) { x.type = movie ? 'movie' : 'series' }
109. release = textOr(movie ? d.release_date : d.first_air_date)
110. if (textOr(x.release) == null && release != null) {
111. x.release = release
112. if (release.length >= 4) { x.year = toNumber(release.slice(0, 4)) }
113. }
114. if (textOr(x.language) == null && textOr(d.original_language) != null) { x.language = d.original_language }
115. // tracker#33 (mission 032): a movie's runtime in minutes (TMDB's 0 = unknown, not stored)
116. if (movie && d.runtime != null && hlTypeName(d.runtime) == 'Number' && d.runtime > 0) { x.runtime = d.runtime }
117. // tracker#21 (merge, mission 070): TMDB's TV type + the kind from the genres just stored (the delta's light step and the
118. // completion of #26 store it too, not only syncShow / the kind backfill)
119. if (!movie && textOr(d.type) != null) {
120. x.tmdbType = d.type
121. x.kind = kindOfTitle(d.type, x.genres)
122. }
123. return x
124. }
126. // THE DAILY SYNC (jobs.hl syncTick, tools/sync-tmdb.hl; tracker#18): after sync.hl `syncShow` (seasons, episodes, poster,
127. // ids, TVmaze) the record's own fields (withDetailsFields) and its cast + crew (#28) from the SAME details answer (`res.details`)
128. // — no extra request. Answers { renamed (the title changed: the search re-indexes it), cast, crew, people, ms, error }
129. static applySynced = (&showId, &d) => {
130. out = { renamed = false cast = 0 crew = 0 people = 0 ms = 0 error = '' }
131. t0 = now()
132. show = showsTable.fetch(showId)
133. if (show == null || d == null) { return out }
134. movie = show.type == 'movie'
135. before = show.title
136. let x = withDetailsFields(show, d, movie)
137. out.renamed = x.title != before
138. if (x.adult != true && hasCredits(d, movie)) {
139. w = withCredits(x, d, movie)
140. x = w.show
141. out.cast = w.cast
142. out.crew = w.crew
143. out.people = w.people
144. }
145. if (showsTable.update(x.id, x) == null) { out.error = x.title + ': record not stored: ' + showsTable.lastError() }
146. out.ms = now() - t0
147. return out
148. }
150. // A SERIES IS DONE IN STEPS (real data: The Late Show, 22 seasons / 4,252 episodes, took 10.7 s as one step — hl:web serves one
151. // request at a time, every page waited): step 1 = the details request (fields + cast); then one step per PART of its seasons
152. // (≤ `partEpisodes` episodes, ≤ 20 seasons = one TMDB request — sync.hl syncShowPart); the last step = ids, TVmaze, poster,
153. // `detailsAt`. A movie is one step. Between two steps every other request is served.
154. // show id → { d (TMDB's answer of step 1), parts ([[season numbers]]), pos, sum } while a series is under way — in memory: a
155. // restart begins again with the details request; what is stored by then is matched by tmdbId, nothing is duplicated
156. static jobs = {}
157. // (TRACKER_DETAILS_PART_EPISODES: the gate sets 1 → one season per step)
158. static partEpisodes = env('TRACKER_DETAILS_PART_EPISODES') != null ? toNumber(env('TRACKER_DETAILS_PART_EPISODES')) : 300
160. static emptySum = () => { return { steps = 0 requests = 0 tvmazeRequests = 0 cast = 0 crew = 0 people = 0 newSeasons = 0 newEpisodes = 0 posters = 0 ids = 0 ms = 0 error = '' } }
161. static addSum = (sum, &r, &ms) => {
162. o = sum
163. o.steps = o.steps + 1
164. o.requests = o.requests + r.requests
165. o.tvmazeRequests = o.tvmazeRequests + r.tvmazeRequests
166. o.newSeasons = o.newSeasons + r.newSeasons + r.tvmazeSeasons
167. o.newEpisodes = o.newEpisodes + r.newEpisodes + r.tvmazeEpisodes
168. o.posters = o.posters + r.posters
169. o.ids = o.ids + r.ids
170. o.ms = o.ms + ms
171. if (r.error != '') { o.error = r.error }
172. return o
173. }
175. // TMDB's season list → parts of ≤ partEpisodes episodes (a bigger season is a part of its own), ≤ 20 seasons each; which of
176. // them are really synced (season 0, empty ones) syncShowPart decides
177. static seasonParts = (&d) => {
178. parts = []
179. let cur = []
180. let n = 0
181. for (x of listOr(d.seasons)) {
182. if (x != null && x.season_number != null) {
183. let c = x.episode_count != null ? x.episode_count : 0
184. if (cur.length > 0 && (n + c > partEpisodes || cur.length == 20)) {
185. parts.push(cur)
186. cur = []
187. n = 0
188. }
189. cur.push(x.season_number)
190. n = n + c
191. }
192. }
193. if (cur.length > 0) { parts.push(cur) }
194. return parts
195. }
197. // a step's answer: { skipped (nothing to do), done, requests / tvmazeRequests (THIS step's — what the pause counts), adult,
198. // error ('' = none), retry (a passing failure: no answer, 429, 5xx), sum (the whole title's counts so far: steps, requests,
199. // cast, newSeasons, newEpisodes, posters, ids, ms) }
200. static stepOut = () => { return { skipped = false done = false requests = 0 tvmazeRequests = 0 adult = null error = '' retry = false sum = emptySum() } }
202. // the last step: the title is complete
203. static markComplete = (&showId, &sum) => {
204. done = showsTable.fetch(showId)
205. done.detailsAt = now()
206. if (sum.error != '') { done.detailsError = sum.error }
207. ok = showsTable.update(done.id, done) != null
208. refreshCatalogShow(done.id)
209. searchTitleChanged(done.id)
210. return ok
211. }
213. // step 1: the details request — the record's fields, the cast; a movie also its ids + poster (done); a series' parts are planned
214. static firstStep = (show) => {
215. out = stepOut()
216. t0 = now()
217. movie = show.type == 'movie'
218. out.requests = 1
219. d = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids,' + (movie ? 'credits' : 'aggregate_credits'))
220. if (d.failed != null) {
221. out.error = show.title + ': ' + d.failed
222. if (d.status == 404) {
223. show.detailsCheck = 'TMDB 404'
224. showsTable.update(show.id, show)
225. out.done = true
226. } else { out.retry = true }
227. return out
228. }
229. out.adult = adultOf(d)
230. let x = withDetailsFields(show, d, movie)
231. // mission 025: a movie has no season/episode counts (hasTvLeftovers)
232. if (movie) { x.seasonsCount = null x.episodesCount = null }
233. if (out.adult == true) {
234. // never public: the flag, no poster, no seasons, no cast (mission 056)
235. x.detailsCheck = 'adult'
236. showsTable.update(x.id, x)
237. refreshCatalogShow(x.id)
238. searchTitleChanged(x.id)
239. out.done = true
240. return out
241. }
242. // tracker#28: the whole cast + the crew, from this same answer
243. w = withCredits(x, d, movie)
244. x = w.show
245. if (showsTable.update(x.id, x) == null) {
246. out.error = show.title + ': not updated: ' + showsTable.lastError()
247. out.retry = true
248. return out
249. }
250. sum = emptySum()
251. sum.cast = w.cast
252. sum.crew = w.crew
253. sum.people = w.people
254. if (movie) {
255. // one request was enough: ids + poster now (syncMovie with this answer)
256. r = syncShowWith(x.id, d)
257. out.requests = r.requests
258. out.sum = addSum(sum, r, now() - t0)
259. if (r.error != '') { out.error = r.error }
260. markComplete(x.id, out.sum)
261. out.done = true
262. return out
263. }
264. sum.steps = 1
265. sum.requests = 1
266. sum.ms = now() - t0
267. // the next steps need the details, not the credits (a long series' aggregate_credits are ~1 MB, copied on every step)
268. dd = d
269. dd.aggregate_credits = null
270. jobs[x.id] = { d = dd parts = seasonParts(d) pos = 0 sum = sum }
271. out.sum = sum
272. return out
273. }
275. // ONE step of completing this title (by our id) — called until `done` (the page's face, the repair job)
276. static completeStep = (&showId) => {
277. show = showsTable.fetch(showId)
278. if (!needsDetails(show)) {
279. jobs[showId] = null
280. let out = stepOut()
281. out.skipped = true
282. out.done = true
283. return out
284. }
285. job = jobs[showId]
286. if (job == null) { return firstStep(show) }
287. t0 = now()
288. let out = stepOut()
289. last = job.pos >= job.parts.length
290. r = syncShowPart(showId, job.d, { seasons = last ? [] : job.parts[job.pos] finish = last })
291. out.requests = r.requests
292. out.tvmazeRequests = r.tvmazeRequests
293. if (r.error != '') { out.error = r.error }
294. j = job
295. j.sum = addSum(j.sum, r, now() - t0)
296. j.pos = j.pos + 1
297. out.sum = j.sum
298. if (!last) {
299. jobs[showId] = j
300. return out
301. }
302. jobs[showId] = null
303. markComplete(showId, j.sum)
304. out.done = true
305. return out
306. }
308. // the line every completed title logs
309. static completeLine = (&title, &sum) => {
310. return '"' + title + '" steps=' + sum.steps + ' requests=' + sum.requests + ' tvmazeRequests=' + sum.tvmazeRequests + ' cast=' + sum.cast + ' newSeasons=' + sum.newSeasons
311. + ' newEpisodes=' + sum.newEpisodes + ' posters=' + sum.posters + ' ids=' + sum.ids + ' ms=' + sum.ms + ' crew=' + sum.crew + ' people=' + sum.people + (sum.error != '' ? ' error: ' + sum.error : '')
312. }
314. // flush shows/seasons/episodes + people (new cast members)
315. static persistDetails = () => {
316. persistSync()
317. personsTable.persist()
318. return true
319. }
321. // THE SEARCH IMPORT, COMPLETE (tracker#14 + #28): search.hl importTitle and, for a title it created, the cast + crew from the
322. // same details answer (credits.hl applyCredits — search.hl cannot call it, this file imports search.hl). Used by the search's
323. // "Add" (components/search.hl) and the curated franchise seed (franchiseseed.hl, tracker#40). → importTitle's answer
324. static importWithCredits = (&kind, tmdbId) => {
325. r = importTitle(kind, tmdbId)
326. if (r.error != null || r.created != true || r.details == null) { return r }
327. c = applyCredits(r.show, r.details)
328. persistDetails()
329. console.log('search import credits: ' + r.path + ' cast=' + c.cast + ' crew=' + c.crew + ' people=' + c.people + ' ms=' + c.ms + (c.error != '' ? ' error: ' + c.error : ''))
330. return r
331. }