tracker

8.9 KB

1. // lib/backfill.hl — THE BACKFILLS of a title's flags (adult, kind; tracker#35 TVmaze's status), one title per step (run by jobs.hl `backfillTick` / `kindTick`, on the
2. // app's clock like the daily sync). Mission 028: out of tmdbsync.hl.
3. //
4. // ADULT FLAG (mission 054): every details answer carries TMDB's `adult` — the sync, the import (search.hl) and the
5. // BACKFILL below store it as show.adult (true/false; null = not known yet, hidden from the public lists, shows.hl
6. // isPublicTitle). THE BACKFILL (`backfillTitle`, run one title per step by jobs.hl `backfillTick`, like the daily sync):
7. // GET <TMDB_BASE_URL>/movie|tv/<tmdbId>?append_to_response=external_ids ONE request: adult, poster_path, imdb/tvdb id
8. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster when its file is missing (not for
9. // an adult title — never shown, mission 056)
10. // for every title whose adult flag is unknown (no TVmaze, no seasons — the daily sync does those for followed shows). A
11. // title TMDB answers 404 for (or one without tmdbId) gets `adultCheck` = why, stays unknown (hidden) and is not asked again.
12. //
13. // KIND (tracker.worldapi.org#21, mission 068): a TV details answer carries TMDB's `type` (Scripted, Miniseries, Documentary,
14. // Reality, Talk Show, News, Video) — the sync, the adult backfill and the import (search.hl) store it as show.tmdbType and the
15. // title's kind ('series' | 'show', shows.hl kindOfTitle) as show.kind. THE KIND BACKFILL (`kindTitle`, run one title per step by
16. // jobs.hl `kindTick`, after the adult backfill, on the same clock):
17. // a title with a show genre (Reality / Talk / News) kind = 'show' from the stored data, NO request
18. // any other TV title without `kind` GET <TMDB_BASE_URL>/tv/<tmdbId> ONE request → tmdbType + kind
19. // A 404 (or no tmdbId) stores the genre's answer ('series') with `kindCheck` = why; it is not asked again. Resumes after a
20. // restart (the queue = every TV title still without `kind`).
21. //
22. // TVMAZE STATUS (tracker#35, mission 030): the sync stores TVmaze's status (Running / Ended / To Be Determined / In Development)
23. // as show.tvmazeStatus whenever it merges TVmaze's episodes (sync.hl, followed series and every completion). THE STATUS BACKFILL
24. // (`tvmazeStatusTitle`, one series per step by jobs.hl `statusTick`, after the details repair, part of that job):
25. // every series with a TVmaze id and no tvmazeStatus GET <TVMAZE_BASE_URL>/shows/<tvmzId> ONE TVmaze request, 550 ms pause
26. // A 404 (or no status in the answer) stores `tvmazeStatusCheck` = why; it is not asked again. Resumes after a restart.
27. import { showsTable, kindOfTitle, hasShowGenre } from './shows.hl'
28. import { textOr, hasId } from './util.hl'
29. import { tmdbGet, externalIdsFor, syncPoster, adultOf, tvmazeShow } from './tmdb.hl'
31. // ---- the adult backfill (mission 054) ------------------------------------------------------------
32. // a title still to ask: its adult flag unknown, it has a tmdbId and TMDB never answered "no such title" for it
33. static needsAdultFlag = (&show) => { return show.adult != true && show.adult != false && show.adultCheck == null && show.tmdbId != null && show.mergedInto == null }
35. // the ids of every title the backfill still has to ask, in table order (read once when the backfill starts — so after a
36. // restart it goes on with what is left)
37. static adultBackfillQueue = () => {
38. out = []
39. for (s of showsTable.find(null, null)) { if (needsAdultFlag(s)) { out.push(s.id) } }
40. return out
41. }
43. // ONE title: one TMDB details request (adult, poster_path, external ids) + its poster when the file is missing. Answers
44. // { requests (TMDB API requests: what the pause counts), posters, adult (true / false / null), ids, errors, error, retry
45. // (a passing failure: no answer, 429, 5xx — ask again later) }. A 404 marks the title `adultCheck` (never asked again).
46. static backfillTitle = (&showId) => {
47. out = { requests = 0 posters = 0 adult = null ids = 0 errors = 0 error = '' retry = false }
48. show = showsTable.fetch(showId)
49. if (show == null || !needsAdultFlag(show)) { return out }
50. movie = show.type == 'movie'
51. out.requests = 1
52. details = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids')
53. if (details.failed != null) {
54. out.errors = 1
55. out.error = show.title + ': ' + details.failed
56. if (details.status == 404) {
57. show.adultCheck = 'TMDB 404'
58. showsTable.update(show.id, show)
59. } else { out.retry = true }
60. return out
61. }
62. ids = externalIdsFor(show, details.external_ids, !movie, false)
63. out.ids = ids.filled
64. out.adult = adultOf(details)
65. // an ADULT title's poster is never shown publicly → not downloaded (mission 056: ~170 MB of the live data); its flag and
66. // ids are stored as for any other title
67. let poster = { saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
68. if (out.adult != true) { poster = syncPoster(show, details.poster_path) }
69. out.posters = poster.saved
70. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
71. if (out.adult == null) { show.adultCheck = 'TMDB gave no adult flag' } else { show.adult = out.adult }
72. if (!movie && textOr(details.type) != null) { show.tmdbType = details.type show.kind = kindOfTitle(details.type, show.genres) }
73. show.imdbId = ids.imdbId
74. show.tvdbId = ids.tvdbId
75. show.image = poster.image
76. show.tmdbPoster = poster.tmdbPoster
77. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
78. return out
79. }
81. // ---- the kind backfill (tracker.worldapi.org#21, mission 068) ----------------------------------------------------
82. // a TV title whose kind is not stored yet
83. static needsKind = (&show) => { return show.type == 'series' && show.kind != 'series' && show.kind != 'show' && show.mergedInto == null }
84. // the ids of every TV title still without a kind, in table order (read once when the backfill starts — it resumes)
85. static kindBackfillQueue = () => {
86. out = []
87. for (s of showsTable.find(null, null)) { if (needsKind(s)) { out.push(s.id) } }
88. return out
89. }
90. // ONE title → { requests (TMDB API requests: what the pause counts), kind ('series' | 'show' | null), fromGenre, errors, error,
91. // retry (a passing failure: no answer, 429, 5xx) }
92. static kindTitle = (&showId) => {
93. out = { requests = 0 kind = null fromGenre = false errors = 0 error = '' retry = false }
94. show = showsTable.fetch(showId)
95. if (show == null || !needsKind(show)) { return out }
96. if (hasShowGenre(show.genres) || show.tmdbId == null) {
97. out.fromGenre = true
98. show.kind = kindOfTitle(null, show.genres)
99. if (!hasShowGenre(show.genres)) { show.kindCheck = 'no tmdbId' }
100. } else {
101. out.requests = 1
102. details = tmdbGet('/tv/' + show.tmdbId)
103. if (details.failed != null) {
104. out.errors = 1
105. out.error = show.title + ': ' + details.failed
106. if (details.status != 404) { out.retry = true return out }
107. show.kindCheck = 'TMDB 404'
108. show.kind = kindOfTitle(null, show.genres)
109. } else {
110. if (textOr(details.type) != null) { show.tmdbType = details.type } else { show.kindCheck = 'TMDB gave no type' }
111. show.kind = kindOfTitle(textOr(details.type), show.genres)
112. }
113. }
114. out.kind = show.kind
115. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
116. return out
117. }
119. // ---- the TVmaze status backfill (tracker#35, mission 030) -------------------------------------------------------------
120. // a live, non-adult series with a TVmaze id whose TVmaze status is not stored and was not given up on
121. static needsTvmazeStatus = (&show) => {
122. return show.type == 'series' && hasId(show.tvmzId) && show.tvmazeStatus == null && show.tvmazeStatusCheck == null && show.adult != true && show.mergedInto == null
123. }
124. // the ids of every such series, in table order (read once when the backfill starts — it resumes)
125. static tvmazeStatusQueue = () => {
126. out = []
127. for (s of showsTable.find(null, null)) { if (needsTvmazeStatus(s)) { out.push(s.id) } }
128. return out
129. }
130. // ONE series → { tvmazeRequests (what the pause counts), status (or null), errors, error, retry (a passing failure) }
131. static tvmazeStatusTitle = (&showId) => {
132. out = { requests = 0 tvmazeRequests = 0 status = null errors = 0 error = '' retry = false }
133. show = showsTable.fetch(showId)
134. if (show == null || !needsTvmazeStatus(show)) { return out }
135. out.tvmazeRequests = 1
136. got = tvmazeShow(show.tvmzId)
137. if (got.failed != null) {
138. out.errors = 1
139. out.error = show.title + ': ' + got.failed
140. if (got.notFound != true) { out.retry = true return out }
141. show.tvmazeStatusCheck = 'TVmaze 404'
142. } else if (got.status == null) { show.tvmazeStatusCheck = 'TVmaze gave no status' } else {
143. show.tvmazeStatus = got.status
144. out.status = got.status
145. }
146. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
147. return out
148. }