tracker

6.6 KB

1. // lib/backfill.hl — THE TWO BACKFILLS of a title's flags, one title per step (run by jobs.hl `backfillTick` / `kindTick`, on the
2. // app's clock like the daily sync). Mission 028: out of tmdbsync.hl.
3. //
4. // ADULT FLAG (mission 054): every details answer carries TMDB's `adult` — the sync, the import (search.hl) and the
5. // BACKFILL below store it as show.adult (true/false; null = not known yet, hidden from the public lists, shows.hl
6. // isPublicTitle). THE BACKFILL (`backfillTitle`, run one title per step by jobs.hl `backfillTick`, like the daily sync):
7. // GET <TMDB_BASE_URL>/movie|tv/<tmdbId>?append_to_response=external_ids ONE request: adult, poster_path, imdb/tvdb id
8. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster when its file is missing (not for
9. // an adult title — never shown, mission 056)
10. // for every title whose adult flag is unknown (no TVmaze, no seasons — the daily sync does those for followed shows). A
11. // title TMDB answers 404 for (or one without tmdbId) gets `adultCheck` = why, stays unknown (hidden) and is not asked again.
12. //
13. // KIND (tracker.worldapi.org#21, mission 068): a TV details answer carries TMDB's `type` (Scripted, Miniseries, Documentary,
14. // Reality, Talk Show, News, Video) — the sync, the adult backfill and the import (search.hl) store it as show.tmdbType and the
15. // title's kind ('series' | 'show', shows.hl kindOfTitle) as show.kind. THE KIND BACKFILL (`kindTitle`, run one title per step by
16. // jobs.hl `kindTick`, after the adult backfill, on the same clock):
17. // a title with a show genre (Reality / Talk / News) kind = 'show' from the stored data, NO request
18. // any other TV title without `kind` GET <TMDB_BASE_URL>/tv/<tmdbId> ONE request → tmdbType + kind
19. // A 404 (or no tmdbId) stores the genre's answer ('series') with `kindCheck` = why; it is not asked again. Resumes after a
20. // restart (the queue = every TV title still without `kind`).
21. import { showsTable, kindOfTitle, hasShowGenre } from './shows.hl'
22. import { textOr } from './util.hl'
23. import { tmdbGet, externalIdsFor, syncPoster, adultOf } from './tmdb.hl'
25. // ---- the adult backfill (mission 054) ------------------------------------------------------------
26. // a title still to ask: its adult flag unknown, it has a tmdbId and TMDB never answered "no such title" for it
27. static needsAdultFlag = (&show) => { return show.adult != true && show.adult != false && show.adultCheck == null && show.tmdbId != null && show.mergedInto == null }
29. // the ids of every title the backfill still has to ask, in table order (read once when the backfill starts — so after a
30. // restart it goes on with what is left)
31. static adultBackfillQueue = () => {
32. out = []
33. for (s of showsTable.find(null, null)) { if (needsAdultFlag(s)) { out.push(s.id) } }
34. return out
35. }
37. // ONE title: one TMDB details request (adult, poster_path, external ids) + its poster when the file is missing. Answers
38. // { requests (TMDB API requests: what the pause counts), posters, adult (true / false / null), ids, errors, error, retry
39. // (a passing failure: no answer, 429, 5xx — ask again later) }. A 404 marks the title `adultCheck` (never asked again).
40. static backfillTitle = (&showId) => {
41. out = { requests = 0 posters = 0 adult = null ids = 0 errors = 0 error = '' retry = false }
42. show = showsTable.fetch(showId)
43. if (show == null || !needsAdultFlag(show)) { return out }
44. movie = show.type == 'movie'
45. out.requests = 1
46. details = tmdbGet((movie ? '/movie/' : '/tv/') + show.tmdbId + '?append_to_response=external_ids')
47. if (details.failed != null) {
48. out.errors = 1
49. out.error = show.title + ': ' + details.failed
50. if (details.status == 404) {
51. show.adultCheck = 'TMDB 404'
52. showsTable.update(show.id, show)
53. } else { out.retry = true }
54. return out
55. }
56. ids = externalIdsFor(show, details.external_ids, !movie, false)
57. out.ids = ids.filled
58. out.adult = adultOf(details)
59. // an ADULT title's poster is never shown publicly → not downloaded (mission 056: ~170 MB of the live data); its flag and
60. // ids are stored as for any other title
61. let poster = { saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
62. if (out.adult != true) { poster = syncPoster(show, details.poster_path) }
63. out.posters = poster.saved
64. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
65. if (out.adult == null) { show.adultCheck = 'TMDB gave no adult flag' } else { show.adult = out.adult }
66. if (!movie && textOr(details.type) != null) { show.tmdbType = details.type show.kind = kindOfTitle(details.type, show.genres) }
67. show.imdbId = ids.imdbId
68. show.tvdbId = ids.tvdbId
69. show.image = poster.image
70. show.tmdbPoster = poster.tmdbPoster
71. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
72. return out
73. }
75. // ---- the kind backfill (tracker.worldapi.org#21, mission 068) ----------------------------------------------------
76. // a TV title whose kind is not stored yet
77. static needsKind = (&show) => { return show.type == 'series' && show.kind != 'series' && show.kind != 'show' && show.mergedInto == null }
78. // the ids of every TV title still without a kind, in table order (read once when the backfill starts — it resumes)
79. static kindBackfillQueue = () => {
80. out = []
81. for (s of showsTable.find(null, null)) { if (needsKind(s)) { out.push(s.id) } }
82. return out
83. }
84. // ONE title → { requests (TMDB API requests: what the pause counts), kind ('series' | 'show' | null), fromGenre, errors, error,
85. // retry (a passing failure: no answer, 429, 5xx) }
86. static kindTitle = (&showId) => {
87. out = { requests = 0 kind = null fromGenre = false errors = 0 error = '' retry = false }
88. show = showsTable.fetch(showId)
89. if (show == null || !needsKind(show)) { return out }
90. if (hasShowGenre(show.genres) || show.tmdbId == null) {
91. out.fromGenre = true
92. show.kind = kindOfTitle(null, show.genres)
93. if (!hasShowGenre(show.genres)) { show.kindCheck = 'no tmdbId' }
94. } else {
95. out.requests = 1
96. details = tmdbGet('/tv/' + show.tmdbId)
97. if (details.failed != null) {
98. out.errors = 1
99. out.error = show.title + ': ' + details.failed
100. if (details.status != 404) { out.retry = true return out }
101. show.kindCheck = 'TMDB 404'
102. show.kind = kindOfTitle(null, show.genres)
103. } else {
104. if (textOr(details.type) != null) { show.tmdbType = details.type } else { show.kindCheck = 'TMDB gave no type' }
105. show.kind = kindOfTitle(textOr(details.type), show.genres)
106. }
107. }
108. out.kind = show.kind
109. if (showsTable.update(show.id, show) == null) { out.errors = out.errors + 1 out.error = show.title + ': not updated: ' + showsTable.lastError() }
110. return out
111. }