tracker

11.6 KB

1. // lib/backfill.hl — THE BACKFILLS of a title's flags (adult, kind; tracker#35 TVmaze's status), one title per step (run by jobs.hl `backfillTick` / `kindTick`, on the
2. // app's clock like the daily sync). Mission 028: out of tmdbsync.hl.
3. //
4. // ADULT FLAG (mission 054): every details answer carries TMDB's `adult` — the sync, the import (search.hl) and the
5. // BACKFILL below store it as show.adult (true/false; null = not known yet, hidden from the public lists, shows.hl
6. // isPublicTitle). THE BACKFILL (`backfillTitle`, run one title per step by jobs.hl `backfillTick`, like the daily sync):
7. // GET <TMDB_BASE_URL>/movie|tv/<tmdbId>?append_to_response=external_ids ONE request: adult, poster_path, imdb/tvdb id
8. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster when its file is missing (not for
9. // an adult title — never shown, mission 056)
10. // for every title whose adult flag is unknown (no TVmaze, no seasons — the daily sync does those for followed shows). A
11. // title TMDB answers 404 for (or one without tmdbId) gets `adultCheck` = why, stays unknown (hidden) and is not asked again.
12. //
13. // KIND (tracker.worldapi.org#21, mission 068): a TV details answer carries TMDB's `type` (Scripted, Miniseries, Documentary,
14. // Reality, Talk Show, News, Video) — the sync, the adult backfill and the import (search.hl) store it as show.tmdbType and the
15. // title's kind ('series' | 'show', shows.hl kindOfTitle) as show.kind. THE KIND BACKFILL (`kindTitle`, run one title per step by
16. // jobs.hl `kindTick`, after the adult backfill, on the same clock):
17. // a title with a show genre (Reality / Talk / News) kind = 'show' from the stored data, NO request
18. // any other TV title without `kind` GET <TMDB_BASE_URL>/tv/<tmdbId> ONE request → tmdbType + kind
19. // A 404 (or no tmdbId) stores the genre's answer ('series') with `kindCheck` = why; it is not asked again. Resumes after a
20. // restart (the queue = every TV title still without `kind`).
21. //
22. // TVMAZE STATUS (tracker#35, mission 030): the sync stores TVmaze's status (Running / Ended / To Be Determined / In Development)
23. // as show.tvmazeStatus whenever it merges TVmaze's episodes (sync.hl, followed series and every completion). THE STATUS BACKFILL
24. // (`tvmazeStatusTitle`, one series per step by jobs.hl `statusTick`, after the details repair, part of that job):
25. // every series with a TVmaze id and no tvmazeStatus GET <TVMAZE_BASE_URL>/shows/<tvmzId> ONE TVmaze request, 550 ms pause
26. // A 404 (or no status in the answer) stores `tvmazeStatusCheck` = why; it is not asked again. Resumes after a restart.
27. //
28. // ART + RUNTIME (tracker#33, mission 032, The Remaining: TMDB has no poster, only a backdrop): THE ART BACKFILL (`artTitle`, one
29. // title per step by jobs.hl `artTick`, one of the last start jobs):
30. // every PUBLIC title without a poster file, every public movie without `runtime` GET <TMDB_BASE_URL>/movie|tv/<tmdbId>
31. // ONE request → the poster (else the backdrop, tmdb.hl syncPoster) + a movie's runtime; `artAt` = asked (never again), a 404
32. // → `artCheck`. A passing failure (no answer, 429, 5xx) is asked again later. Resumes after a restart.
33. import { now } from 'hl:time'
34. import { showsTable, kindOfTitle, hasShowGenre, isPublicTitle, posterUrlOf } from './shows.hl'
35. import { textOr, hasId } from './util.hl'
36. import { tmdbGet, externalIdsFor, syncPoster, adultOf, tvmazeShow } from './tmdb.hl'
38. // ---- the adult backfill (mission 054) ------------------------------------------------------------
39. // a title still to ask: its adult flag unknown, it has a tmdbId and TMDB never answered "no such title" for it
40. static needsAdultFlag = (&show) => { return show.adult != true && show.adult != false && show.adultCheck == null && show.tmdbId != null && show.mergedInto == null }
42. // the ids of every title the backfill still has to ask, in table order (read once when the backfill starts — so after a
43. // restart it goes on with what is left)
44. static adultBackfillQueue = () => {
45. out = []
46. for (s of showsTable.find(null, null)) { if (needsAdultFlag(s)) { out.push(s.id) } }
47. return out
48. }
50. // ONE title: one TMDB details request (adult, poster_path, external ids) + its poster when the file is missing. Answers
51. // { requests (TMDB API requests: what the pause counts), posters, adult (true / false / null), ids, errors, error, retry
52. // (a passing failure: no answer, 429, 5xx — ask again later) }. A 404 marks the title `adultCheck` (never asked again).
53. static backfillTitle = (&showId) => {
54. out = { requests = 0 posters = 0 adult = null ids = 0 errors = 0 error = '' retry = false }
55. show = showsTable.fetch(showId)
56. if (show == null || !needsAdultFlag(show)) { return out }
57. movie = show.type == 'movie'
58. out.requests = 1
59. details = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids')
60. if (details.failed != null) {
61. out.errors = 1
62. out.error = show.title + ': ' + details.failed
63. if (details.status == 404) {
64. show.adultCheck = 'TMDB 404'
65. showsTable.update(show.id, show)
66. } else { out.retry = true }
67. return out
68. }
69. ids = externalIdsFor(show, details.external_ids, !movie, false)
70. out.ids = ids.filled
71. out.adult = adultOf(details)
72. // an ADULT title's poster is never shown publicly → not downloaded (mission 056: ~170 MB of the live data); its flag and
73. // ids are stored as for any other title
74. let poster = { saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
75. if (out.adult != true) { poster = syncPoster(show, details.poster_path, details.backdrop_path) }
76. out.posters = poster.saved
77. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
78. if (out.adult == null) { show.adultCheck = 'TMDB gave no adult flag' } else { show.adult = out.adult }
79. if (!movie && textOr(details.type) != null) { show.tmdbType = details.type show.kind = kindOfTitle(details.type, show.genres) }
80. show.imdbId = ids.imdbId
81. show.tvdbId = ids.tvdbId
82. show.image = poster.image
83. show.tmdbPoster = poster.tmdbPoster
84. show.posterFromBackdrop = poster.fromBackdrop
85. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
86. return out
87. }
89. // ---- the kind backfill (tracker.worldapi.org#21, mission 068) ----------------------------------------------------
90. // a TV title whose kind is not stored yet
91. static needsKind = (&show) => { return show.type == 'series' && show.kind != 'series' && show.kind != 'show' && show.mergedInto == null }
92. // the ids of every TV title still without a kind, in table order (read once when the backfill starts — it resumes)
93. static kindBackfillQueue = () => {
94. out = []
95. for (s of showsTable.find(null, null)) { if (needsKind(s)) { out.push(s.id) } }
96. return out
97. }
98. // ONE title → { requests (TMDB API requests: what the pause counts), kind ('series' | 'show' | null), fromGenre, errors, error,
99. // retry (a passing failure: no answer, 429, 5xx) }
100. static kindTitle = (&showId) => {
101. out = { requests = 0 kind = null fromGenre = false errors = 0 error = '' retry = false }
102. show = showsTable.fetch(showId)
103. if (show == null || !needsKind(show)) { return out }
104. if (hasShowGenre(show.genres) || show.tmdbId == null) {
105. out.fromGenre = true
106. show.kind = kindOfTitle(null, show.genres)
107. if (!hasShowGenre(show.genres)) { show.kindCheck = 'no tmdbId' }
108. } else {
109. out.requests = 1
110. details = tmdbGet('/tv/' + show.tmdbId)
111. if (details.failed != null) {
112. out.errors = 1
113. out.error = show.title + ': ' + details.failed
114. if (details.status != 404) { out.retry = true return out }
115. show.kindCheck = 'TMDB 404'
116. show.kind = kindOfTitle(null, show.genres)
117. } else {
118. if (textOr(details.type) != null) { show.tmdbType = details.type } else { show.kindCheck = 'TMDB gave no type' }
119. show.kind = kindOfTitle(textOr(details.type), show.genres)
120. }
121. }
122. out.kind = show.kind
123. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
124. return out
125. }
127. // ---- the TVmaze status backfill (tracker#35, mission 030) -------------------------------------------------------------
128. // a live, non-adult series with a TVmaze id whose TVmaze status is not stored and was not given up on
129. static needsTvmazeStatus = (&show) => {
130. return show.type == 'series' && hasId(show.tvmzId) && show.tvmazeStatus == null && show.tvmazeStatusCheck == null && show.adult != true && show.mergedInto == null
131. }
132. // the ids of every such series, in table order (read once when the backfill starts — it resumes)
133. static tvmazeStatusQueue = () => {
134. out = []
135. for (s of showsTable.find(null, null)) { if (needsTvmazeStatus(s)) { out.push(s.id) } }
136. return out
137. }
138. // ONE series → { tvmazeRequests (what the pause counts), status (or null), errors, error, retry (a passing failure) }
139. static tvmazeStatusTitle = (&showId) => {
140. out = { requests = 0 tvmazeRequests = 0 status = null errors = 0 error = '' retry = false }
141. show = showsTable.fetch(showId)
142. if (show == null || !needsTvmazeStatus(show)) { return out }
143. out.tvmazeRequests = 1
144. got = tvmazeShow(show.tvmzId)
145. if (got.failed != null) {
146. out.errors = 1
147. out.error = show.title + ': ' + got.failed
148. if (got.notFound != true) { out.retry = true return out }
149. show.tvmazeStatusCheck = 'TVmaze 404'
150. } else if (got.status == null) { show.tvmazeStatusCheck = 'TVmaze gave no status' } else {
151. show.tvmazeStatus = got.status
152. out.status = got.status
153. }
154. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
155. return out
156. }
158. // ---- the art backfill (tracker#33, mission 032) ---------------------------------------------------------------------------
159. // a public title we never asked for its art: no poster file, or a movie without runtime
160. static needsArt = (&show) => {
161. if (show.artAt != null || show.artCheck != null || show.tmdbId == null || !isPublicTitle(show)) { return false }
162. if (show.type == 'movie' && show.runtime == null) { return true }
163. return (show.type == 'movie' || show.type == 'series') && posterUrlOf(show) == '/posters/none'
164. }
165. static artQueue = () => {
166. out = []
167. for (s of showsTable.iterate(null, null)) { if (needsArt(s)) { out.push(s.id) } }
168. return out
169. }
170. // ONE title → { requests (TMDB requests: what the pause counts), posters, backdrops, runtime (stored or null), errors, error, retry }
171. static artTitle = (&showId) => {
172. out = { requests = 0 posters = 0 backdrops = 0 runtime = null errors = 0 error = '' retry = false }
173. show = showsTable.fetch(showId)
174. if (show == null || !needsArt(show)) { return out }
175. movie = show.type == 'movie'
176. out.requests = 1
177. details = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId)
178. if (details.failed != null) {
179. out.errors = 1
180. out.error = show.title + ': ' + details.failed
181. if (details.status != 404) { out.retry = true return out }
182. show.artCheck = 'TMDB 404'
183. showsTable.update(show.id, show)
184. return out
185. }
186. poster = syncPoster(show, details.poster_path, details.backdrop_path)
187. out.requests = out.requests + poster.requests
188. if (poster.saved > 0) { if (poster.fromBackdrop) { out.backdrops = 1 } else { out.posters = 1 } }
189. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
190. show.image = poster.image
191. show.tmdbPoster = poster.tmdbPoster
192. show.posterFromBackdrop = poster.fromBackdrop
193. if (movie && details.runtime != null && hlTypeName(details.runtime) == 'Number' && details.runtime > 0) {
194. show.runtime = details.runtime
195. out.runtime = details.runtime
196. }
197. show.artAt = now()
198. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
199. return out
200. }