tracker

9.0 KB

1. // lib/tmdb.hl — TALKING TO TMDB AND TVMAZE (tracker.worldapi.org#9/#12/#15): the one GET with the token, the TVmaze lookups, the
2. // external ids a title may take from them, TMDB's adult flag, the poster download, and the pacing every runner keeps between
3. // two titles. Shared by the sync (sync.hl), the backfills (backfill.hl), the completion and the credits (details.hl, credits.hl),
4. // the person fill (people.hl), the search (search.hl), the daily run's change lists (dailysync.hl) and the collection seed
5. // (collections.hl). Until mission 028 this was the top half of tmdbsync.hl.
6. //
7. // Env: TMDB_READ_TOKEN (required unless TMDB_BASE_URL points at a fake), TMDB_BASE_URL
8. // (https://api.themoviedb.org/3), TMDB_IMAGE_URL (https://image.tmdb.org/t/p) — the gate points both at its
9. // fake TMDB (tests/browser.mjs).
10. import { fetch } from 'hl:fetch'
11. import { mkDir, exists, writeFile } from 'hl:fs'
12. import { envOr, postersDir, textOr, hasId } from './util.hl'
13. import { stripHtml } from './sync-helpers.hl'
15. static tmdbBase = envOr('TMDB_BASE_URL', 'https://api.themoviedb.org/3')
16. static tmdbImageBase = envOr('TMDB_IMAGE_URL', 'https://image.tmdb.org/t/p')
17. static tmdbToken = envOr('TMDB_READ_TOKEN', '')
19. static tvmazeBase = envOr('TVMAZE_BASE_URL', 'https://api.tvmaze.com')
21. // the sync can run: a token is set (never printed) — the dev .env on Loreana has one, a gate sets its own
22. static syncEnabled = tmdbToken != ''
24. // TMDB allows 40 requests / 10 s: every runner waits this long per request it made before the next show
25. static msPerRequest = 260
26. // TVmaze allows 20 requests / 10 s (its API docs): 550 ms per lookup made
27. static msPerTvmazeRequest = 550
28. // how long a runner waits after a show before the next one (both runners: jobs.hl syncTick, tools/sync-tmdb.hl)
29. static pauseMsAfter = (&r) => { return r.requests * msPerRequest + r.tvmazeRequests * msPerTvmazeRequest }
31. // one GET → the parsed JSON, or { failed = '<why>' }. A refused/timed-out fetch is an Error event (absorbed
32. // by the runner's `on Error`), the fetch then yields null.
33. static tmdbGet = (&path) => {
34. r = fetch(tmdbBase + path, { headers = { 'authorization' = 'Bearer ' + tmdbToken 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
35. if (r == null || r.status == null || r.status == 0) { return { failed = 'no answer for ' + path } }
36. if (r.status != 200) { return { failed = 'HTTP ' + r.status + ' for ' + path status = r.status } }
37. j = r.json()
38. if (j == null) { return { failed = 'no JSON for ' + path } }
39. return j
40. }
42. // ---- external ids (tracker.worldapi.org#12) -----------------------------------------------------
43. // an IMDb id as TMDB gives it ('tt' + digits), else null
44. static imdbOr = (&v) => {
45. t = textOr(v)
46. if (t == null || t.length < 3 || t.slice(0, 2) != 'tt') { return null }
47. let i = 2
48. while (i < t.length) {
49. if (!'0123456789'.includes(t[i])) { return null }
50. i = i + 1
51. }
52. return t
53. }
54. // a positive numeric id (TMDB's tvdb_id, TVmaze's id), else null
55. static numIdOr = (&v) => { return v != null && hlTypeName(v) == 'Number' && v > 0 ? v : null }
57. // TVmaze's lookup by one external id: { id } (null = TVmaze doesn't know it, 404) or { failed }. hl:fetch follows the
58. // 301 to /shows/<id> by itself, so a hit answers the show's own JSON.
59. static tvmazeLookup = (&param, &value) => {
60. r = fetch(tvmazeBase + '/lookup/shows?' + param + '=' + value, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
61. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for ' + param + '=' + value } }
62. if (r.status == 404) { return { id = null } }
63. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for ' + param + '=' + value } }
64. j = r.json()
65. if (j == null || numIdOr(j.id) == null) { return { failed = 'TVmaze: no id for ' + param + '=' + value } }
66. return { id = j.id }
67. }
69. // the ids to SET on a show from TMDB's external_ids (+ TVmaze for a series): { imdbId, tvdbId, tvmzId } — each the
70. // show's own value when it has one (never overwritten), else the found one, else null — plus { filled, tvmazeRequests,
71. // failed }. TVmaze is asked by imdb, then by thetvdb, only while the show (after TMDB) still has no tvmzId — and only
72. // when `askTvmaze` (the adult backfill does not: one request per title)
73. static externalIdsFor = (&show, &ext, &isSeries, &askTvmaze) => {
74. out = { imdbId = show.imdbId tvdbId = show.tvdbId tvmzId = show.tvmzId filled = 0 tvmazeRequests = 0 failed = null }
75. e = ext != null ? ext : {}
76. if (!hasId(out.imdbId) && imdbOr(e.imdb_id) != null) { out.imdbId = e.imdb_id out.filled = out.filled + 1 }
77. if (!isSeries) { return out }
78. if (!hasId(out.tvdbId) && numIdOr(e.tvdb_id) != null) { out.tvdbId = e.tvdb_id out.filled = out.filled + 1 }
79. if (hasId(out.tvmzId) || !askTvmaze) { return out }
80. tries = []
81. if (imdbOr('' + out.imdbId) != null) { tries.push({ param = 'imdb' value = '' + out.imdbId }) }
82. if (numIdOr(out.tvdbId) != null) { tries.push({ param = 'thetvdb' value = '' + out.tvdbId }) }
83. for (t of tries) {
84. if (!hasId(out.tvmzId) && out.failed == null) {
85. out.tvmazeRequests = out.tvmazeRequests + 1
86. let found = tvmazeLookup(t.param, t.value)
87. if (found.failed != null) { out.failed = found.failed } else if (found.id != null) { out.tvmzId = found.id out.filled = out.filled + 1 }
88. }
89. }
90. return out
91. }
93. // TVmaze's show WITH its episode list, one request (tracker#18: `?embed=episodes` brings the show's own summary too — before
94. // it was `/shows/<id>/episodes`): { list, summary (HTML stripped, or null), status (tracker#35: Running / Ended / To Be
95. // Determined / In Development, or null) } or { failed }
96. static tvmazeEpisodes = (&tvmzId) => {
97. r = fetch(tvmazeBase + '/shows/' + tvmzId + '?embed=episodes', { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
98. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for episodes of ' + tvmzId } }
99. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for episodes of ' + tvmzId } }
100. j = r.json()
101. if (j == null || j._embedded == null || j._embedded.episodes == null || j._embedded.episodes.length == null) { return { failed = 'TVmaze: no episode list for ' + tvmzId } }
102. return { list = j._embedded.episodes summary = stripHtml(j.summary) status = textOr(j.status) }
103. }
105. // tracker#35: TVmaze's show alone (`GET /shows/<id>`, one request) — { status (or null) }, { notFound = true } (404) or { failed }
106. // (no answer, 429, 5xx: ask again later). The status backfill (backfill.hl) asks it for a series the sync has not brought one for.
107. static tvmazeShow = (&tvmzId) => {
108. r = fetch(tvmazeBase + '/shows/' + tvmzId, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
109. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for show ' + tvmzId } }
110. if (r.status == 404) { return { notFound = true failed = 'TVmaze: HTTP 404 for show ' + tvmzId } }
111. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for show ' + tvmzId } }
112. j = r.json()
113. if (j == null) { return { failed = 'TVmaze: no JSON for show ' + tvmzId } }
114. return { status = textOr(j.status) }
115. }
117. // ---- posters ----------------------------------------------------------------------------------
118. // the extension of a TMDB poster_path ('/abc.jpg' → 'jpg'); only the ones the poster route knows
119. static posterExt = (&posterPath) => {
120. parts = posterPath.split('.')
121. ext = parts[parts.length - 1].toLowerCase()
122. if (ext == 'jpg' || ext == 'jpeg' || ext == 'png' || ext == 'webp') { return ext }
123. return null
124. }
126. // downloads the w342 poster when the file is missing or TMDB has a new one; answers { requests, saved, failed }
127. // and sets show.image / show.tmdbPoster on the (caller's) record — the file is <oldId>.<ext>, the name
128. // shows.hl posterName() already builds, served by images.hl (/posters/:name)
129. static syncPoster = (&show, posterPath) => {
130. out = { requests = 0 saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
131. if (textOr(posterPath) == null || show.oldId == null) { return out }
132. ext = posterExt(posterPath)
133. if (ext == null) { return out }
134. file = postersDir + '/' + show.oldId + '.' + ext
135. if (exists(file) && show.tmdbPoster == posterPath && show.image == ext) { return out }
136. out.requests = 1
137. r = fetch(tmdbImageBase + '/w342' + posterPath, { headers = { 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
138. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
139. out.failed = 'poster ' + posterPath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
140. return out
141. }
142. mkDir(postersDir)
143. writeFile(file, toBytes(r.body), 420)
144. out.saved = 1
145. out.image = ext
146. out.tmdbPoster = posterPath
147. return out
148. }
150. // TMDB's adult flag of a details/search answer: true / false, null when it has none (mission 054)
151. static adultOf = (&j) => {
152. if (j == null) { return null }
153. if (j.adult == true) { return true }
154. if (j.adult == false) { return false }
155. return null
156. }