tracker

11.9 KB

1. // details.hl — A TITLE'S FULL DETAILS (tracker.worldapi.org#26). Creator: "when i go to an artist and it loads the bibliography
2. // those loaded movies and series miss most information and it stays that way when i click on them".
3. //
4. // The person page's fill (people.hl, #16) adds every title we don't have as a MINIMAL record — title, year, overview, genres,
5. // type, tmdbId, adult, this one person as cast; no poster, no seasons, no external ids, no homepage/tagline/status
6. // (`minimal = true`). A search import (search.hl importTitle) is full except for the cast. Here a minimal title gets what a
7. // full import has, plus its cast:
8. //
9. // GET <TMDB>/movie/<tmdbId>?append_to_response=external_ids,credits a movie
10. // GET <TMDB>/tv/<tmdbId>?append_to_response=external_ids,aggregate_credits a series (cast of ALL seasons, roles merged)
11. // then tmdbsync.hl `syncShowPart(id, <that answer>, …)` — the daily sync's work, without asking for the details again, IN
12. // STEPS (see `jobs` below): a series' seasons + episodes (`season/1,…`), then the poster (w342), missing imdb/tvdb ids,
13. // TVmaze lookup + its episode merge (#15); a movie its imdb id + poster
14. //
15. // Stored: overview (`summary` only when ours is empty; `tmdbSummary`), homepage, tagline, status, genres (by TMDB id, when
16. // TMDB gives any), release/year/language when we have none, `adult`, the CAST (TMDB's first `castLimit`, in TMDB's order;
17. // a person we have by TMDB id is linked, a new one is added — people.hl castPersonId; cast entries we had and TMDB does not
18. // list are kept after them). Then `detailsAt` (ms) = COMPLETE, never asked again (the daily sync keeps a FOLLOWED title fresh).
19. // TMDB 404 → `detailsCheck = 'TMDB 404'` (never asked again; the page shows what it has). TMDB says ADULT → the flag is
20. // stored, `detailsCheck = 'adult'`, no poster/seasons (mission 056: an adult title is never public, its poster not kept).
21. //
22. // WHO CALLS IT (`completeStep`, again and again until `done`): the show page when an incomplete title is opened
23. // (components/show.hl face `showComplete`, skeleton meanwhile) and the repair job in project.hl (`repairTick`, one step per tick
24. // on the app's clock, like the adult backfill) for every incomplete title that exists when the app starts. Adult titles are
25. // left alone by both.
26. import { now } from 'hl:time'
27. import { env } from 'hl:proc'
28. import { showsTable, personsTable } from './shows.hl'
29. import { tmdbGet, textOr, adultOf, syncShowWith, syncShowPart, persistSync } from './tmdbsync.hl'
30. import { searchTitleChanged } from './search.hl'
31. import { refreshCatalogShow } from './catalog.hl'
32. import { castPersonId, genreIdsOfTmdb } from './people.hl'
34. // how many of TMDB's cast a title keeps (TMDB's own order = billing; a movie can list 100+, each would be a new person)
35. static castLimit = 20
37. static listOr = (v) => { return v == null ? [] : v }
39. // INCOMPLETE = a filmography title (`minimal`) or an import whose sync never finished (`imported`, no `tmdbSync`), not
40. // completed (`detailsAt`) and not given up on (`detailsCheck`)
41. static isIncomplete = (s) => {
42. return s != null && s.tmdbId != null && s.detailsAt == null && s.detailsCheck == null && (s.minimal == true || (s.imported != null && s.tmdbSync == null))
43. }
44. // what the show page and the repair complete: an incomplete title that is not adult (mission 054/056: never public)
45. static needsDetails = (s) => { return isIncomplete(s) && s.adult != true }
47. // every title the repair job has to do, in table order (read once when it starts — so a restart goes on with what is left)
48. static detailsRepairQueue = () => {
49. let out = []
50. for (s of showsTable.find(null, null)) { if (needsDetails(s)) { out.push(s.id) } }
51. return out
52. }
54. // a series' roles ([{ character, episode_count }]) as one text: "Mother / Lamia"
55. static rolesText = (roles) => {
56. let out = ''
57. for (r of listOr(roles)) {
58. let c = r != null ? textOr(r.character) : null
59. if (c != null && !(' / ' + out + ' / ').includes(' / ' + c + ' / ')) { out = out == '' ? c : out + ' / ' + c }
60. }
61. return out
62. }
64. // TMDB's cast → [{ tmdbId, name, character, gender }], TMDB's order, at most castLimit, adult people left out
65. static castOf = (d, movie) => {
66. let src = movie ? (d.credits != null ? d.credits.cast : null) : (d.aggregate_credits != null ? d.aggregate_credits.cast : null)
67. let out = []
68. for (c of listOr(src)) {
69. if (out.length < castLimit && c != null && c.id != null && hlTypeName(c.id) == 'Number' && textOr(c.name) != null && c.adult != true) {
70. let role = movie ? textOr(c.character) : rolesText(c.roles)
71. out.push({ tmdbId = c.id name = c.name character = role != null ? role : '' gender = c.gender })
72. }
73. }
74. return out
75. }
77. // the title's cast: TMDB's (linked to our people, new ones added) first, then the entries we had that TMDB does not list
78. static mergedCast = (show, tmdbCast) => {
79. let out = []
80. let have = {}
81. let showKey = show.id
82. for (c of tmdbCast) {
83. let pid = castPersonId(c, showKey)
84. if (pid != null && have[pid] != true) {
85. have[pid] = true
86. out.push({ person = pid character = c.character actor = c.name })
87. }
88. }
89. for (c of listOr(show.cast)) {
90. if (c != null && c.person != null && have[c.person] != true) {
91. have[c.person] = true
92. out.push(c)
93. }
94. }
95. return out
96. }
98. // A SERIES IS DONE IN STEPS (real data: The Late Show, 22 seasons / 4,252 episodes, took 10.7 s as one step — hl:web serves one
99. // request at a time, every page waited): step 1 = the details request (fields + cast); then one step per PART of its seasons
100. // (≤ `partEpisodes` episodes, ≤ 20 seasons = one TMDB request — tmdbsync.hl syncShowPart); the last step = ids, TVmaze, poster,
101. // `detailsAt`. A movie is one step. Between two steps every other request is served.
102. // show id → { d (TMDB's answer of step 1), parts ([[season numbers]]), pos, sum } while a series is under way — in memory: a
103. // restart begins again with the details request; what is stored by then is matched by tmdbId, nothing is duplicated
104. static jobs = {}
105. // (TRACKER_DETAILS_PART_EPISODES: the gate sets 1 → one season per step)
106. static partEpisodes = env('TRACKER_DETAILS_PART_EPISODES') != null ? toNumber(env('TRACKER_DETAILS_PART_EPISODES')) : 300
108. static emptySum = () => { return { steps = 0 requests = 0 tvmazeRequests = 0 cast = 0 newSeasons = 0 newEpisodes = 0 posters = 0 ids = 0 ms = 0 error = '' } }
109. static addSum = (sum, r, ms) => {
110. let o = sum
111. o.steps = o.steps + 1
112. o.requests = o.requests + r.requests
113. o.tvmazeRequests = o.tvmazeRequests + r.tvmazeRequests
114. o.newSeasons = o.newSeasons + r.newSeasons + r.tvmazeSeasons
115. o.newEpisodes = o.newEpisodes + r.newEpisodes + r.tvmazeEpisodes
116. o.posters = o.posters + r.posters
117. o.ids = o.ids + r.ids
118. o.ms = o.ms + ms
119. if (r.error != '') { o.error = r.error }
120. return o
121. }
123. // TMDB's season list → parts of ≤ partEpisodes episodes (a bigger season is a part of its own), ≤ 20 seasons each; which of
124. // them are really synced (season 0, empty ones) syncShowPart decides
125. static seasonParts = (d) => {
126. let parts = []
127. let cur = []
128. let n = 0
129. for (x of listOr(d.seasons)) {
130. if (x != null && x.season_number != null) {
131. let c = x.episode_count != null ? x.episode_count : 0
132. if (cur.length > 0 && (n + c > partEpisodes || cur.length == 20)) {
133. parts.push(cur)
134. cur = []
135. n = 0
136. }
137. cur.push(x.season_number)
138. n = n + c
139. }
140. }
141. if (cur.length > 0) { parts.push(cur) }
142. return parts
143. }
145. // a step's answer: { skipped (nothing to do), done, requests / tvmazeRequests (THIS step's — what the pause counts), adult,
146. // error ('' = none), retry (a passing failure: no answer, 429, 5xx), sum (the whole title's counts so far: steps, requests,
147. // cast, newSeasons, newEpisodes, posters, ids, ms) }
148. static stepOut = () => { return { skipped = false done = false requests = 0 tvmazeRequests = 0 adult = null error = '' retry = false sum = emptySum() } }
150. // the last step: the title is complete
151. static markComplete = (showId, sum) => {
152. let done = showsTable.fetch(showId)
153. done.detailsAt = now()
154. if (sum.error != '') { done.detailsError = sum.error }
155. let ok = showsTable.update(done.id, done) != null
156. refreshCatalogShow(done.id)
157. searchTitleChanged(done.id)
158. return ok
159. }
161. // step 1: the details request — the record's fields, the cast; a movie also its ids + poster (done); a series' parts are planned
162. static firstStep = (show) => {
163. let out = stepOut()
164. let t0 = now()
165. let movie = show.type == 'movie'
166. out.requests = 1
167. let d = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids,' + (movie ? 'credits' : 'aggregate_credits'))
168. if (d.failed != null) {
169. out.error = show.title + ': ' + d.failed
170. if (d.status == 404) {
171. show.detailsCheck = 'TMDB 404'
172. showsTable.update(show.id, show)
173. out.done = true
174. } else { out.retry = true }
175. return out
176. }
177. let x = show
178. out.adult = adultOf(d)
179. if (out.adult != null) { x.adult = out.adult }
180. let overview = textOr(d.overview)
181. if (overview != null) {
182. x.tmdbSummary = overview
183. if (textOr(x.summary) == null) { x.summary = overview }
184. }
185. if (textOr(d.homepage) != null) { x.homepage = d.homepage }
186. if (textOr(d.tagline) != null) { x.tagline = d.tagline }
187. if (textOr(d.status) != null) { x.status = d.status }
188. let tmdbGenreIds = []
189. for (g of listOr(d.genres)) { if (g != null && g.id != null) { tmdbGenreIds.push(g.id) } }
190. let genres = genreIdsOfTmdb(tmdbGenreIds)
191. if (genres.length > 0) { x.genres = genres }
192. let release = textOr(movie ? d.release_date : d.first_air_date)
193. if (textOr(x.release) == null && release != null) {
194. x.release = release
195. if (release.length >= 4) { x.year = toNumber(release.slice(0, 4)) }
196. }
197. if (textOr(x.language) == null && textOr(d.original_language) != null) { x.language = d.original_language }
198. if (out.adult == true) {
199. // never public: the flag, no poster, no seasons, no cast (mission 056)
200. x.detailsCheck = 'adult'
201. showsTable.update(x.id, x)
202. refreshCatalogShow(x.id)
203. searchTitleChanged(x.id)
204. out.done = true
205. return out
206. }
207. let tmdbCast = castOf(d, movie)
208. x.cast = mergedCast(x, tmdbCast)
209. if (showsTable.update(x.id, x) == null) {
210. out.error = show.title + ': not updated: ' + showsTable.lastError()
211. out.retry = true
212. return out
213. }
214. let sum = emptySum()
215. sum.cast = tmdbCast.length
216. if (movie) {
217. // one request was enough: ids + poster now (syncMovie with this answer)
218. let r = syncShowWith(x.id, d)
219. out.requests = r.requests
220. out.sum = addSum(sum, r, now() - t0)
221. if (r.error != '') { out.error = r.error }
222. markComplete(x.id, out.sum)
223. out.done = true
224. return out
225. }
226. sum.steps = 1
227. sum.requests = 1
228. sum.ms = now() - t0
229. jobs[x.id] = { d = d parts = seasonParts(d) pos = 0 sum = sum }
230. out.sum = sum
231. return out
232. }
234. // ONE step of completing this title (by our id) — called until `done` (the page's face, the repair job)
235. static completeStep = (showId) => {
236. let show = showsTable.fetch(showId)
237. if (!needsDetails(show)) {
238. jobs[showId] = null
239. let out = stepOut()
240. out.skipped = true
241. out.done = true
242. return out
243. }
244. let job = jobs[showId]
245. if (job == null) { return firstStep(show) }
246. let t0 = now()
247. let out = stepOut()
248. let last = job.pos >= job.parts.length
249. let r = syncShowPart(showId, job.d, { seasons = last ? [] : job.parts[job.pos] finish = last })
250. out.requests = r.requests
251. out.tvmazeRequests = r.tvmazeRequests
252. if (r.error != '') { out.error = r.error }
253. let j = job
254. j.sum = addSum(j.sum, r, now() - t0)
255. j.pos = j.pos + 1
256. out.sum = j.sum
257. if (!last) {
258. jobs[showId] = j
259. return out
260. }
261. jobs[showId] = null
262. markComplete(showId, j.sum)
263. out.done = true
264. return out
265. }
267. // the line every completed title logs
268. static completeLine = (title, sum) => {
269. return '"' + title + '" steps=' + sum.steps + ' requests=' + sum.requests + ' tvmazeRequests=' + sum.tvmazeRequests + ' cast=' + sum.cast + ' newSeasons=' + sum.newSeasons
270. + ' newEpisodes=' + sum.newEpisodes + ' posters=' + sum.posters + ' ids=' + sum.ids + ' ms=' + sum.ms + (sum.error != '' ? ' error: ' + sum.error : '')
271. }
273. // flush shows/seasons/episodes + people (new cast members)
274. static persistDetails = () => {
275. persistSync()
276. personsTable.persist()
277. return true
278. }