tracker

8.1 KB

1. // lib/tmdb.hl — TALKING TO TMDB AND TVMAZE (tracker.worldapi.org#9/#12/#15): the one GET with the token, the TVmaze lookups, the
2. // external ids a title may take from them, TMDB's adult flag, the poster download, and the pacing every runner keeps between
3. // two titles. Shared by the sync (sync.hl), the backfills (backfill.hl), the completion and the credits (details.hl, credits.hl),
4. // the person fill (people.hl), the search (search.hl), the daily run's change lists (dailysync.hl) and the collection seed
5. // (collections.hl). Until mission 028 this was the top half of tmdbsync.hl.
6. //
7. // Env: TMDB_READ_TOKEN (required unless TMDB_BASE_URL points at a fake), TMDB_BASE_URL
8. // (https://api.themoviedb.org/3), TMDB_IMAGE_URL (https://image.tmdb.org/t/p) — the gate points both at its
9. // fake TMDB (tests/browser.mjs).
10. import { fetch } from 'hl:fetch'
11. import { mkDir, exists, writeFile } from 'hl:fs'
12. import { envOr, postersDir, textOr, hasId } from './util.hl'
13. import { stripHtml } from './sync-helpers.hl'
15. static tmdbBase = envOr('TMDB_BASE_URL', 'https://api.themoviedb.org/3')
16. static tmdbImageBase = envOr('TMDB_IMAGE_URL', 'https://image.tmdb.org/t/p')
17. static tmdbToken = envOr('TMDB_READ_TOKEN', '')
19. static tvmazeBase = envOr('TVMAZE_BASE_URL', 'https://api.tvmaze.com')
21. // the sync can run: a token is set (never printed) — the dev .env on Loreana has one, a gate sets its own
22. static syncEnabled = tmdbToken != ''
24. // TMDB allows 40 requests / 10 s: every runner waits this long per request it made before the next show
25. static msPerRequest = 260
26. // TVmaze allows 20 requests / 10 s (its API docs): 550 ms per lookup made
27. static msPerTvmazeRequest = 550
28. // how long a runner waits after a show before the next one (both runners: project.hl syncTick, tools/sync-tmdb.hl)
29. static pauseMsAfter = (&r) => { return r.requests * msPerRequest + r.tvmazeRequests * msPerTvmazeRequest }
31. // one GET → the parsed JSON, or { failed = '<why>' }. A refused/timed-out fetch is an Error event (absorbed
32. // by the runner's `on Error`), the fetch then yields null.
33. static tmdbGet = (&path) => {
34. let r = fetch(tmdbBase + path, { headers = { 'authorization' = 'Bearer ' + tmdbToken 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
35. if (r == null || r.status == null || r.status == 0) { return { failed = 'no answer for ' + path } }
36. if (r.status != 200) { return { failed = 'HTTP ' + r.status + ' for ' + path status = r.status } }
37. let j = r.json()
38. if (j == null) { return { failed = 'no JSON for ' + path } }
39. return j
40. }
42. // ---- external ids (tracker.worldapi.org#12) -----------------------------------------------------
43. // an IMDb id as TMDB gives it ('tt' + digits), else null
44. static imdbOr = (&v) => {
45. let t = textOr(v)
46. if (t == null || t.length < 3 || t.slice(0, 2) != 'tt') { return null }
47. let i = 2
48. while (i < t.length) {
49. if (!'0123456789'.includes(t[i])) { return null }
50. i = i + 1
51. }
52. return t
53. }
54. // a positive numeric id (TMDB's tvdb_id, TVmaze's id), else null
55. static numIdOr = (&v) => { return v != null && hlTypeName(v) == 'Number' && v > 0 ? v : null }
57. // TVmaze's lookup by one external id: { id } (null = TVmaze doesn't know it, 404) or { failed }. hl:fetch follows the
58. // 301 to /shows/<id> by itself, so a hit answers the show's own JSON.
59. static tvmazeLookup = (&param, &value) => {
60. let r = fetch(tvmazeBase + '/lookup/shows?' + param + '=' + value, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
61. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for ' + param + '=' + value } }
62. if (r.status == 404) { return { id = null } }
63. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for ' + param + '=' + value } }
64. let j = r.json()
65. if (j == null || numIdOr(j.id) == null) { return { failed = 'TVmaze: no id for ' + param + '=' + value } }
66. return { id = j.id }
67. }
69. // the ids to SET on a show from TMDB's external_ids (+ TVmaze for a series): { imdbId, tvdbId, tvmzId } — each the
70. // show's own value when it has one (never overwritten), else the found one, else null — plus { filled, tvmazeRequests,
71. // failed }. TVmaze is asked by imdb, then by thetvdb, only while the show (after TMDB) still has no tvmzId — and only
72. // when `askTvmaze` (the adult backfill does not: one request per title)
73. static externalIdsFor = (&show, &ext, &isSeries, &askTvmaze) => {
74. let out = { imdbId = show.imdbId tvdbId = show.tvdbId tvmzId = show.tvmzId filled = 0 tvmazeRequests = 0 failed = null }
75. let e = ext != null ? ext : {}
76. if (!hasId(out.imdbId) && imdbOr(e.imdb_id) != null) { out.imdbId = e.imdb_id out.filled = out.filled + 1 }
77. if (!isSeries) { return out }
78. if (!hasId(out.tvdbId) && numIdOr(e.tvdb_id) != null) { out.tvdbId = e.tvdb_id out.filled = out.filled + 1 }
79. if (hasId(out.tvmzId) || !askTvmaze) { return out }
80. let tries = []
81. if (imdbOr('' + out.imdbId) != null) { tries.push({ param = 'imdb' value = '' + out.imdbId }) }
82. if (numIdOr(out.tvdbId) != null) { tries.push({ param = 'thetvdb' value = '' + out.tvdbId }) }
83. for (t of tries) {
84. if (!hasId(out.tvmzId) && out.failed == null) {
85. out.tvmazeRequests = out.tvmazeRequests + 1
86. let found = tvmazeLookup(t.param, t.value)
87. if (found.failed != null) { out.failed = found.failed } else if (found.id != null) { out.tvmzId = found.id out.filled = out.filled + 1 }
88. }
89. }
90. return out
91. }
93. // TVmaze's show WITH its episode list, one request (tracker#18: `?embed=episodes` brings the show's own summary too — before
94. // it was `/shows/<id>/episodes`): { list, summary (HTML stripped, or null) } or { failed }
95. static tvmazeEpisodes = (&tvmzId) => {
96. let r = fetch(tvmazeBase + '/shows/' + tvmzId + '?embed=episodes', { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
97. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for episodes of ' + tvmzId } }
98. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for episodes of ' + tvmzId } }
99. let j = r.json()
100. if (j == null || j._embedded == null || j._embedded.episodes == null || j._embedded.episodes.length == null) { return { failed = 'TVmaze: no episode list for ' + tvmzId } }
101. return { list = j._embedded.episodes summary = stripHtml(j.summary) }
102. }
104. // ---- posters ----------------------------------------------------------------------------------
105. // the extension of a TMDB poster_path ('/abc.jpg' → 'jpg'); only the ones the poster route knows
106. static posterExt = (&posterPath) => {
107. let parts = posterPath.split('.')
108. let ext = parts[parts.length - 1].toLowerCase()
109. if (ext == 'jpg' || ext == 'jpeg' || ext == 'png' || ext == 'webp') { return ext }
110. return null
111. }
113. // downloads the w342 poster when the file is missing or TMDB has a new one; answers { requests, saved, failed }
114. // and sets show.image / show.tmdbPoster on the (caller's) record — the file is <oldId>.<ext>, the name
115. // shows.hl posterName() already builds, served by project.hl's /posters/:name route
116. static syncPoster = (&show, posterPath) => {
117. let out = { requests = 0 saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
118. if (textOr(posterPath) == null || show.oldId == null) { return out }
119. let ext = posterExt(posterPath)
120. if (ext == null) { return out }
121. let file = postersDir + '/' + show.oldId + '.' + ext
122. if (exists(file) && show.tmdbPoster == posterPath && show.image == ext) { return out }
123. out.requests = 1
124. let r = fetch(tmdbImageBase + '/w342' + posterPath, { headers = { 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
125. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
126. out.failed = 'poster ' + posterPath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
127. return out
128. }
129. mkDir(postersDir)
130. writeFile(file, toBytes(r.body), 420)
131. out.saved = 1
132. out.image = ext
133. out.tmdbPoster = posterPath
134. return out
135. }
137. // TMDB's adult flag of a details/search answer: true / false, null when it has none (mission 054)
138. static adultOf = (&j) => {
139. if (j == null) { return null }
140. if (j.adult == true) { return true }
141. if (j.adult == false) { return false }
142. return null
143. }