tracker

14.6 KB

1. // lib/details.hl — A TITLE'S FULL DETAILS (tracker.worldapi.org#26). Creator: "when i go to an artist and it loads the bibliography
2. // those loaded movies and series miss most information and it stays that way when i click on them".
3. //
4. // The person page's fill (people.hl, #16) adds every title we don't have as a MINIMAL record — title, year, overview, genres,
5. // type, tmdbId, adult, this one person as cast; no poster, no seasons, no external ids, no homepage/tagline/status
6. // (`minimal = true`). A search import (search.hl importTitle) is full except for the cast. Here a minimal title gets what a
7. // full import has, plus its cast:
8. //
9. // GET <TMDB>/movie/<tmdbId>?append_to_response=external_ids,credits a movie
10. // GET <TMDB>/tv/<tmdbId>?append_to_response=external_ids,aggregate_credits a series (cast of ALL seasons, roles merged)
11. // then sync.hl `syncShowPart(id, <that answer>, …)` — the daily sync's work, without asking for the details again, IN
12. // STEPS (see `jobs` below): a series' seasons + episodes (`season/1,…`), then the poster (w342), missing imdb/tvdb ids,
13. // TVmaze lookup + its episode merge (#15); a movie its imdb id + poster
14. //
15. // Stored (`withDetailsFields`, shared with the daily sync since tracker#18): title, overview → `tmdbSummary` (`summary` is the
16. // creator's own text, never written), homepage, tagline, status, genres (by TMDB id, when
17. // TMDB gives any), release/year/language when we have none, `adult`, the CAST (TMDB's first `castLimit`, in TMDB's order;
18. // a person we have by TMDB id is linked, a new one is added — people.hl castPersonId; cast entries we had and TMDB does not
19. // list are kept after them). Then `detailsAt` (ms) = COMPLETE, never asked again (the daily sync keeps a FOLLOWED title fresh).
20. // TMDB 404 → `detailsCheck = 'TMDB 404'` (never asked again; the page shows what it has). TMDB says ADULT → the flag is
21. // stored, `detailsCheck = 'adult'`, no poster/seasons (mission 056: an adult title is never public, its poster not kept).
22. //
23. // tracker#33 (mission 029): INCOMPLETE is every title TMDB's details never went through (shows.hl isIncomplete) — also the old
24. // tracker's migrated rows (5,697 on the live copy: never synced, 691 series without seasons); they are completed the same way.
25. //
26. // WHO CALLS IT (`completeStep`, again and again until `done`): the show page when an incomplete title is opened
27. // (components/show.hl face `showComplete`, skeleton meanwhile) and the repair job in jobs.hl (`repairTick`, one step per tick
28. // on the app's clock, like the adult backfill) for every incomplete title that exists when the app starts. Adult titles are
29. // left alone by both.
30. //
31. // The cast and crew of the same answer: credits.hl (withCredits). Mission 028: the cast + crew parts moved there, `isIncomplete`
32. // to shows.hl.
33. import { now } from 'hl:time'
34. import { env } from 'hl:proc'
35. import { textOr, listOr } from './util.hl'
36. import { showsTable, personsTable, kindOfTitle, isIncomplete, missingParts } from './shows.hl'
37. import { tmdbGet, adultOf } from './tmdb.hl'
38. import { syncShowWith, syncShowPart, persistSync } from './sync.hl'
39. import { searchTitleChanged } from './search.hl'
40. import { refreshCatalogShow } from './catalog.hl'
41. import { genreIdsOfTmdb } from './people.hl'
42. import { withCredits } from './credits.hl'
43. import { hasCredits } from './credits-helpers.hl'
45. // what the show page and the repair complete: an incomplete title that is not adult (mission 054/056: never public)
46. static needsDetails = (&s) => { return isIncomplete(s) && s.adult != true && s.mergedInto == null }
48. // every title the repair job has to do (read once when it starts — so a restart goes on with what is left). tracker#33: the ones
49. // a page shows as reduced first (shows.hl missingParts: no seasons, poster, genres …), then the rest, each in table order
50. static detailsRepairQueue = () => {
51. first = []
52. rest = []
53. for (s of showsTable.find(null, null)) {
54. if (needsDetails(s)) {
55. if (missingParts(s).length > 0) { first.push(s.id) } else { rest.push(s.id) }
56. }
57. }
58. for (id of rest) { first.push(id) }
59. return first
60. }
62. // THE RECORD'S OWN FIELDS from a details answer (tracker#26 completion; tracker#18 the daily sync too — creator: "some shows changed
63. // descriptions and titles and that did not update"): title (a series' `name`, a movie's `title`), TMDB's overview → `tmdbSummary`,
64. // homepage, tagline, status, genres (by TMDB id), adult; type / release / year / language only when we have none. A TMDB value
65. // that is empty never overwrites ours. `summary` is the creator's OWN text — never written here (the page shows summary >
66. // tmdbSummary > tvmazeSummary, shows.hl summaryOf). The slug (`urlSegment`) never changes with the title.
67. // mission 070 (conductor): the sync never DESTROYS what the old tracker's migration brought — before TMDB's value replaces a
68. // migrated record's title / genres / homepage / tagline for the FIRST time, the old one is kept in `migratedTitle` /
69. // `migratedGenres` / `migratedHomepage` / `migratedTagline` (kept, shown nowhere; set once, a later rename leaves it). A record is
70. // migrated when its `oldId` is the old tracker's (not `tmdb-…`: search import, filmography fill). `status`, counts, ids and the
71. // poster are TMDB's state, written by the sync since #9 — not kept.
72. static isMigratedRecord = (&show) => { return show.oldId != null && ('' + show.oldId).slice(0, 5) != 'tmdb-' }
73. static sameIdList = (&a, &b) => {
74. x = listOr(a)
75. y = listOr(b)
76. if (x.length != y.length) { return false }
77. let i = 0
78. while (i < x.length) { if ('' + x[i] != '' + y[i]) { return false } i = i + 1 }
79. return true
80. }
81. static withDetailsFields = (show, &d, &movie) => {
82. x = show
83. migrated = isMigratedRecord(show)
84. adult = adultOf(d)
85. if (adult != null) { x.adult = adult }
86. let title = textOr(movie ? d.title : d.name)
87. if (title != null) {
88. if (migrated && x.migratedTitle == null && textOr(x.title) != null && x.title != title) { x.migratedTitle = x.title }
89. x.title = title
90. }
91. if (textOr(d.overview) != null) { x.tmdbSummary = d.overview }
92. if (textOr(d.homepage) != null) {
93. if (migrated && x.migratedHomepage == null && textOr(x.homepage) != null && x.homepage != d.homepage) { x.migratedHomepage = x.homepage }
94. x.homepage = d.homepage
95. }
96. if (textOr(d.tagline) != null) {
97. if (migrated && x.migratedTagline == null && textOr(x.tagline) != null && x.tagline != d.tagline) { x.migratedTagline = x.tagline }
98. x.tagline = d.tagline
99. }
100. if (textOr(d.status) != null) { x.status = d.status }
101. tmdbGenreIds = []
102. for (g of listOr(d.genres)) { if (g != null && g.id != null) { tmdbGenreIds.push(g.id) } }
103. genres = genreIdsOfTmdb(tmdbGenreIds)
104. if (genres.length > 0) {
105. if (migrated && x.migratedGenres == null && listOr(x.genres).length > 0 && !sameIdList(x.genres, genres)) { x.migratedGenres = x.genres }
106. x.genres = genres
107. }
108. if (textOr(x.type) == null) { x.type = movie ? 'movie' : 'series' }
109. release = textOr(movie ? d.release_date : d.first_air_date)
110. if (textOr(x.release) == null && release != null) {
111. x.release = release
112. if (release.length >= 4) { x.year = toNumber(release.slice(0, 4)) }
113. }
114. if (textOr(x.language) == null && textOr(d.original_language) != null) { x.language = d.original_language }
115. // tracker#21 (merge, mission 070): TMDB's TV type + the kind from the genres just stored (the delta's light step and the
116. // completion of #26 store it too, not only syncShow / the kind backfill)
117. if (!movie && textOr(d.type) != null) {
118. x.tmdbType = d.type
119. x.kind = kindOfTitle(d.type, x.genres)
120. }
121. return x
122. }
124. // THE DAILY SYNC (jobs.hl syncTick, tools/sync-tmdb.hl; tracker#18): after sync.hl `syncShow` (seasons, episodes, poster,
125. // ids, TVmaze) the record's own fields (withDetailsFields) and its cast + crew (#28) from the SAME details answer (`res.details`)
126. // — no extra request. Answers { renamed (the title changed: the search re-indexes it), cast, crew, people, ms, error }
127. static applySynced = (&showId, &d) => {
128. out = { renamed = false cast = 0 crew = 0 people = 0 ms = 0 error = '' }
129. t0 = now()
130. show = showsTable.fetch(showId)
131. if (show == null || d == null) { return out }
132. movie = show.type == 'movie'
133. before = show.title
134. let x = withDetailsFields(show, d, movie)
135. out.renamed = x.title != before
136. if (x.adult != true && hasCredits(d, movie)) {
137. w = withCredits(x, d, movie)
138. x = w.show
139. out.cast = w.cast
140. out.crew = w.crew
141. out.people = w.people
142. }
143. if (showsTable.update(x.id, x) == null) { out.error = x.title + ': record not stored: ' + showsTable.lastError() }
144. out.ms = now() - t0
145. return out
146. }
148. // A SERIES IS DONE IN STEPS (real data: The Late Show, 22 seasons / 4,252 episodes, took 10.7 s as one step — hl:web serves one
149. // request at a time, every page waited): step 1 = the details request (fields + cast); then one step per PART of its seasons
150. // (≤ `partEpisodes` episodes, ≤ 20 seasons = one TMDB request — sync.hl syncShowPart); the last step = ids, TVmaze, poster,
151. // `detailsAt`. A movie is one step. Between two steps every other request is served.
152. // show id → { d (TMDB's answer of step 1), parts ([[season numbers]]), pos, sum } while a series is under way — in memory: a
153. // restart begins again with the details request; what is stored by then is matched by tmdbId, nothing is duplicated
154. static jobs = {}
155. // (TRACKER_DETAILS_PART_EPISODES: the gate sets 1 → one season per step)
156. static partEpisodes = env('TRACKER_DETAILS_PART_EPISODES') != null ? toNumber(env('TRACKER_DETAILS_PART_EPISODES')) : 300
158. static emptySum = () => { return { steps = 0 requests = 0 tvmazeRequests = 0 cast = 0 crew = 0 people = 0 newSeasons = 0 newEpisodes = 0 posters = 0 ids = 0 ms = 0 error = '' } }
159. static addSum = (sum, &r, &ms) => {
160. o = sum
161. o.steps = o.steps + 1
162. o.requests = o.requests + r.requests
163. o.tvmazeRequests = o.tvmazeRequests + r.tvmazeRequests
164. o.newSeasons = o.newSeasons + r.newSeasons + r.tvmazeSeasons
165. o.newEpisodes = o.newEpisodes + r.newEpisodes + r.tvmazeEpisodes
166. o.posters = o.posters + r.posters
167. o.ids = o.ids + r.ids
168. o.ms = o.ms + ms
169. if (r.error != '') { o.error = r.error }
170. return o
171. }
173. // TMDB's season list → parts of ≤ partEpisodes episodes (a bigger season is a part of its own), ≤ 20 seasons each; which of
174. // them are really synced (season 0, empty ones) syncShowPart decides
175. static seasonParts = (&d) => {
176. parts = []
177. let cur = []
178. let n = 0
179. for (x of listOr(d.seasons)) {
180. if (x != null && x.season_number != null) {
181. let c = x.episode_count != null ? x.episode_count : 0
182. if (cur.length > 0 && (n + c > partEpisodes || cur.length == 20)) {
183. parts.push(cur)
184. cur = []
185. n = 0
186. }
187. cur.push(x.season_number)
188. n = n + c
189. }
190. }
191. if (cur.length > 0) { parts.push(cur) }
192. return parts
193. }
195. // a step's answer: { skipped (nothing to do), done, requests / tvmazeRequests (THIS step's — what the pause counts), adult,
196. // error ('' = none), retry (a passing failure: no answer, 429, 5xx), sum (the whole title's counts so far: steps, requests,
197. // cast, newSeasons, newEpisodes, posters, ids, ms) }
198. static stepOut = () => { return { skipped = false done = false requests = 0 tvmazeRequests = 0 adult = null error = '' retry = false sum = emptySum() } }
200. // the last step: the title is complete
201. static markComplete = (&showId, &sum) => {
202. done = showsTable.fetch(showId)
203. done.detailsAt = now()
204. if (sum.error != '') { done.detailsError = sum.error }
205. ok = showsTable.update(done.id, done) != null
206. refreshCatalogShow(done.id)
207. searchTitleChanged(done.id)
208. return ok
209. }
211. // step 1: the details request — the record's fields, the cast; a movie also its ids + poster (done); a series' parts are planned
212. static firstStep = (show) => {
213. out = stepOut()
214. t0 = now()
215. movie = show.type == 'movie'
216. out.requests = 1
217. d = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids,' + (movie ? 'credits' : 'aggregate_credits'))
218. if (d.failed != null) {
219. out.error = show.title + ': ' + d.failed
220. if (d.status == 404) {
221. show.detailsCheck = 'TMDB 404'
222. showsTable.update(show.id, show)
223. out.done = true
224. } else { out.retry = true }
225. return out
226. }
227. out.adult = adultOf(d)
228. let x = withDetailsFields(show, d, movie)
229. // mission 025: a movie has no season/episode counts (hasTvLeftovers)
230. if (movie) { x.seasonsCount = null x.episodesCount = null }
231. if (out.adult == true) {
232. // never public: the flag, no poster, no seasons, no cast (mission 056)
233. x.detailsCheck = 'adult'
234. showsTable.update(x.id, x)
235. refreshCatalogShow(x.id)
236. searchTitleChanged(x.id)
237. out.done = true
238. return out
239. }
240. // tracker#28: the whole cast + the crew, from this same answer
241. w = withCredits(x, d, movie)
242. x = w.show
243. if (showsTable.update(x.id, x) == null) {
244. out.error = show.title + ': not updated: ' + showsTable.lastError()
245. out.retry = true
246. return out
247. }
248. sum = emptySum()
249. sum.cast = w.cast
250. sum.crew = w.crew
251. sum.people = w.people
252. if (movie) {
253. // one request was enough: ids + poster now (syncMovie with this answer)
254. r = syncShowWith(x.id, d)
255. out.requests = r.requests
256. out.sum = addSum(sum, r, now() - t0)
257. if (r.error != '') { out.error = r.error }
258. markComplete(x.id, out.sum)
259. out.done = true
260. return out
261. }
262. sum.steps = 1
263. sum.requests = 1
264. sum.ms = now() - t0
265. // the next steps need the details, not the credits (a long series' aggregate_credits are ~1 MB, copied on every step)
266. dd = d
267. dd.aggregate_credits = null
268. jobs[x.id] = { d = dd parts = seasonParts(d) pos = 0 sum = sum }
269. out.sum = sum
270. return out
271. }
273. // ONE step of completing this title (by our id) — called until `done` (the page's face, the repair job)
274. static completeStep = (&showId) => {
275. show = showsTable.fetch(showId)
276. if (!needsDetails(show)) {
277. jobs[showId] = null
278. let out = stepOut()
279. out.skipped = true
280. out.done = true
281. return out
282. }
283. job = jobs[showId]
284. if (job == null) { return firstStep(show) }
285. t0 = now()
286. let out = stepOut()
287. last = job.pos >= job.parts.length
288. r = syncShowPart(showId, job.d, { seasons = last ? [] : job.parts[job.pos] finish = last })
289. out.requests = r.requests
290. out.tvmazeRequests = r.tvmazeRequests
291. if (r.error != '') { out.error = r.error }
292. j = job
293. j.sum = addSum(j.sum, r, now() - t0)
294. j.pos = j.pos + 1
295. out.sum = j.sum
296. if (!last) {
297. jobs[showId] = j
298. return out
299. }
300. jobs[showId] = null
301. markComplete(showId, j.sum)
302. out.done = true
303. return out
304. }
306. // the line every completed title logs
307. static completeLine = (&title, &sum) => {
308. return '"' + title + '" steps=' + sum.steps + ' requests=' + sum.requests + ' tvmazeRequests=' + sum.tvmazeRequests + ' cast=' + sum.cast + ' newSeasons=' + sum.newSeasons
309. + ' newEpisodes=' + sum.newEpisodes + ' posters=' + sum.posters + ' ids=' + sum.ids + ' ms=' + sum.ms + ' crew=' + sum.crew + ' people=' + sum.people + (sum.error != '' ? ' error: ' + sum.error : '')
310. }
312. // flush shows/seasons/episodes + people (new cast members)
313. static persistDetails = () => {
314. persistSync()
315. personsTable.persist()
316. return true
317. }