tracker

14.0 KB

1. // tmdbsync.hl — THE TMDB SYNC (tracker.worldapi.org#9): for every show SOMEONE follows, fetch the show's
2. // details, seasons and episodes from TMDB and bring storage/mpackdb/ up to date, plus the show's poster.
3. // Used by BOTH runners: the daily run inside the app (project.hl `syncTick`, one show per step) and the
4. // command-line tool (tools/sync-tmdb.hl). Reference for the API use: the old tracker's
5. // crates/sync/commands/sync.js (read-only).
6. //
7. // GET <TMDB_BASE_URL>/tv/<tmdbId> the details (status, counts,
8. // overview, poster_path, seasons[])
9. // GET <TMDB_BASE_URL>/tv/<tmdbId>?append_to_response=season/1,season/2… up to 20 seasons with their
10. // episodes in ONE request
11. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster → storage/mpackdb/posters/
12. // Authorization: Bearer $TMDB_READ_TOKEN
13. //
14. // RULES (ticket): new seasons/episodes are ADDED, existing ones UPDATED (title, summary, release) —
15. // matched by tmdbId, never duplicated; NOTHING is deleted. An episode without a tmdbId (161 migrated ones)
16. // is matched by its S/E number instead and gets the tmdbId. A season (no tmdbId in the migrated data) is
17. // matched by (show, seasonNumber). A TMDB value that is empty/null never overwrites a stored one.
18. // Season 0 ("Specials") is only synced for a show that already has a season 0 (else every followed show
19. // would suddenly flood /my/unwatched with specials); a TMDB season with no episodes is not created.
20. //
21. // IDS ARE SELF-ASSIGNED (randomBytes, like watches.hl): hl:mpackdb's own generated @id can repeat within
22. // one millisecond, which is what made update() lose rows (hybriel#113, fixed upstream, not in the vendored
23. // build) — a table holding only unique ids is safe to update().
24. //
25. // ONE PROCESS PER STORAGE: hl:mpackdb does not coordinate two processes (hybriel#21) — the in-app run uses
26. // the app's own open tables (imported from shows.hl); the tool only ever runs against a stopped app or a copy.
27. //
28. // Env: TMDB_READ_TOKEN (required unless TMDB_BASE_URL points at a fake), TMDB_BASE_URL
29. // (https://api.themoviedb.org/3), TMDB_IMAGE_URL (https://image.tmdb.org/t/p) — the gate points both at its
30. // fake TMDB (tests/browser.mjs).
31. import { fetch } from 'hl:fetch'
32. import { env } from 'hl:proc'
33. import { now } from 'hl:time'
34. import { randomBytes } from 'hl:crypto'
35. import { writeFile, mkDir, exists } from 'hl:fs'
36. import { showsTable, seasonsTable, episodesTable, storageDir } from './shows.hl'
38. static envOr = (name, fallback) => {
39. let v = env(name)
40. return v != null && v.trim() != '' ? v.trim() : fallback
41. }
42. static tmdbBase = envOr('TMDB_BASE_URL', 'https://api.themoviedb.org/3')
43. static tmdbImageBase = envOr('TMDB_IMAGE_URL', 'https://image.tmdb.org/t/p')
44. static tmdbToken = envOr('TMDB_READ_TOKEN', '')
45. static postersDir = storageDir + '/posters'
47. // the sync can run: a token is set (never printed) — the dev .env on Loreana has one, a gate sets its own
48. static syncEnabled = tmdbToken != ''
50. // TMDB allows 40 requests / 10 s: every runner waits this long per request it made before the next show
51. static msPerRequest = 260
53. // one GET → the parsed JSON, or { failed = '<why>' }. A refused/timed-out fetch is an Error event (absorbed
54. // by the runner's `on Error`), the fetch then yields null.
55. static tmdbGet = (path) => {
56. let r = fetch(tmdbBase + path, { headers = { 'authorization' = 'Bearer ' + tmdbToken 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
57. if (r == null || r.status == null || r.status == 0) { return { failed = 'no answer for ' + path } }
58. if (r.status != 200) { return { failed = 'HTTP ' + r.status + ' for ' + path } }
59. let j = r.json()
60. if (j == null) { return { failed = 'no JSON for ' + path } }
61. return j
62. }
64. static textOr = (v) => { return v != null && hlTypeName(v) == 'String' && v.trim() != '' ? v : null }
66. // ---- posters ----------------------------------------------------------------------------------
67. // the extension of a TMDB poster_path ('/abc.jpg' → 'jpg'); only the ones the poster route knows
68. static posterExt = (posterPath) => {
69. let parts = posterPath.split('.')
70. let ext = parts[parts.length - 1].toLowerCase()
71. if (ext == 'jpg' || ext == 'jpeg' || ext == 'png' || ext == 'webp') { return ext }
72. return null
73. }
75. // downloads the w342 poster when the file is missing or TMDB has a new one; answers { requests, saved, failed }
76. // and sets show.image / show.tmdbPoster on the (caller's) record — the file is <oldId>.<ext>, the name
77. // shows.hl posterName() already builds, served by project.hl's /posters/:name route
78. static syncPoster = (show, posterPath) => {
79. let out = { requests = 0 saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
80. if (textOr(posterPath) == null || show.oldId == null) { return out }
81. let ext = posterExt(posterPath)
82. if (ext == null) { return out }
83. let file = postersDir + '/' + show.oldId + '.' + ext
84. if (exists(file) && show.tmdbPoster == posterPath && show.image == ext) { return out }
85. out.requests = 1
86. let r = fetch(tmdbImageBase + '/w342' + posterPath, { headers = { 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
87. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
88. out.failed = 'poster ' + posterPath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
89. return out
90. }
91. mkDir(postersDir)
92. writeFile(file, toBytes(r.body), 420)
93. out.saved = 1
94. out.image = ext
95. out.tmdbPoster = posterPath
96. return out
97. }
99. // ---- one show -----------------------------------------------------------------------------------
100. static newId = () => { return randomBytes(8, 'hex') }
102. // which TMDB seasons to sync: every numbered one with episodes; 0 only when the show has a season 0 already
103. static wantedSeasonNumbers = (details, localByNumber) => {
104. let out = []
105. if (details.seasons == null) { return out }
106. for (s of details.seasons) {
107. let n = s.season_number
108. if (n != null && (n > 0 || localByNumber['' + n] != null) && (s.episode_count == null || s.episode_count > 0 || localByNumber['' + n] != null)) { out.push(n) }
109. }
110. return out
111. }
113. // the seasons in batches of 20 (TMDB's append_to_response limit)
114. static batchesOf = (list, size) => {
115. let out = []
116. let cur = []
117. for (x of list) {
118. cur.push(x)
119. if (cur.length == size) { out.push(cur) cur = [] }
120. }
121. if (cur.length > 0) { out.push(cur) }
122. return out
123. }
125. // fields of an episode/season from TMDB that differ from the stored record (null/empty never overwrites)
126. static changedText = (stored, fresh) => { return fresh != null && fresh != stored }
128. // Sync ONE show (by our id). Answers counts: { requests, newSeasons, newEpisodes, updatedEpisodes,
129. // updatedSeasons, posters, errors, error }. Never throws for a TMDB failure — it is counted and the show left as is.
130. static syncShow = (showId) => {
131. let res = { requests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 errors = 0 error = '' }
132. let show = showsTable.fetch(showId)
133. if (show == null) { res.errors = 1 res.error = 'no show ' + showId return res }
134. if (show.tmdbId == null) { res.errors = 1 res.error = show.title + ': no tmdbId' return res }
136. res.requests = 1
137. let details = tmdbGet('/tv/' + show.tmdbId)
138. if (details.failed != null) { res.errors = 1 res.error = show.title + ': ' + details.failed return res }
140. // what we have: seasons by number, episodes by tmdbId and by S/E number
141. let seasonsByNumber = {}
142. let epByTmdb = {}
143. let epByNumber = {}
144. for (sid of show.seasons) {
145. let s = seasonsTable.fetch(sid)
146. if (s != null) {
147. seasonsByNumber['' + s.seasonNumber] = s
148. for (eid of s.episodes) {
149. let e = episodesTable.fetch(eid)
150. if (e != null) {
151. if (e.tmdbId != null) { epByTmdb['' + e.tmdbId] = e }
152. epByNumber[e.seasonNumber + 'x' + e.episodeNumber] = e
153. }
154. }
155. }
156. }
158. // plain locals for the record literals below: inside a literal, an entry named `show`/`season` would shadow
159. // the variable for every later entry (SPEC "A literal's entries are evaluated in order")
160. let showKey = show.id
161. let showTmdb = show.tmdbId
162. let showSeasons = []
163. for (sid of show.seasons) { showSeasons.push(sid) }
164. let showChanged = false
166. for (batch of batchesOf(wantedSeasonNumbers(details, seasonsByNumber), 20)) {
167. let names = []
168. for (n of batch) { names.push('season/' + n) }
169. res.requests = res.requests + 1
170. let withSeasons = tmdbGet('/tv/' + show.tmdbId + '?append_to_response=' + names.join(','))
171. if (withSeasons.failed != null) {
172. res.errors = res.errors + 1
173. res.error = show.title + ': ' + withSeasons.failed
174. } else {
175. for (n of batch) {
176. let ts = withSeasons['season/' + n]
177. if (ts != null && ts.episodes != null) {
178. let season = seasonsByNumber['' + n]
179. let seasonIsNew = season == null
180. let seasonChanged = false
181. if (seasonIsNew) {
182. season = { id = newId() seasonNumber = n title = textOr(ts.name) != null ? ts.name : 'Season ' + n
183. release = textOr(ts.air_date) summary = textOr(ts.overview) != null ? ts.overview : ''
184. show = showKey episodes = [] oldId = 'tmdb-season:' + showTmdb + ':' + n }
185. } else {
186. if (changedText(season.title, textOr(ts.name))) { season.title = ts.name seasonChanged = true }
187. if (changedText(season.summary, textOr(ts.overview))) { season.summary = ts.overview seasonChanged = true }
188. if (changedText(season.release, textOr(ts.air_date))) { season.release = ts.air_date seasonChanged = true }
189. }
190. let seasonKey = season.id
191. let seasonEpisodes = []
192. for (eid of season.episodes) { seasonEpisodes.push(eid) }
193. for (te of ts.episodes) {
194. if (te.id != null && te.episode_number != null) {
195. let e = epByTmdb['' + te.id]
196. if (e == null) {
197. let byNum = epByNumber[n + 'x' + te.episode_number]
198. if (byNum != null && byNum.tmdbId == null) { e = byNum }
199. }
200. if (e == null) {
201. let fresh = { id = newId() episodeNumber = te.episode_number seasonNumber = n
202. title = textOr(te.name) summary = textOr(te.overview) != null ? te.overview : '' release = textOr(te.air_date)
203. show = showKey season = seasonKey tmdbId = te.id oldId = 'tmdb-episode:' + te.id }
204. let putId = episodesTable.put(fresh)
205. if (putId == null) {
206. res.errors = res.errors + 1
207. res.error = show.title + ': episode ' + te.id + ' not stored: ' + episodesTable.lastError()
208. } else {
209. seasonEpisodes.push(fresh.id)
210. epByTmdb['' + te.id] = fresh
211. res.newEpisodes = res.newEpisodes + 1
212. seasonChanged = true
213. }
214. } else {
215. let changed = false
216. if (e.tmdbId == null) { e.tmdbId = te.id changed = true }
217. if (changedText(e.title, textOr(te.name))) { e.title = te.name changed = true }
218. if (changedText(e.summary, textOr(te.overview))) { e.summary = te.overview changed = true }
219. if (changedText(e.release, textOr(te.air_date))) { e.release = te.air_date changed = true }
220. if (changed) {
221. if (episodesTable.update(e.id, e) == null) {
222. res.errors = res.errors + 1
223. res.error = show.title + ': episode ' + e.id + ' not updated: ' + episodesTable.lastError()
224. } else {
225. epByTmdb['' + e.tmdbId] = e
226. res.updatedEpisodes = res.updatedEpisodes + 1
227. }
228. }
229. }
230. }
231. }
232. season.episodes = seasonEpisodes
233. if (seasonIsNew) {
234. if (seasonsTable.put(season) == null) {
235. res.errors = res.errors + 1
236. res.error = show.title + ': season ' + n + ' not stored: ' + seasonsTable.lastError()
237. } else {
238. seasonsByNumber['' + n] = season
239. showSeasons.push(season.id)
240. showChanged = true
241. res.newSeasons = res.newSeasons + 1
242. }
243. } else if (seasonChanged) {
244. if (seasonsTable.update(season.id, season) == null) {
245. res.errors = res.errors + 1
246. res.error = show.title + ': season ' + season.id + ' not updated: ' + seasonsTable.lastError()
247. } else {
248. res.updatedSeasons = res.updatedSeasons + 1
249. }
250. }
251. }
252. }
253. }
254. }
256. // the show itself: status, counts, TMDB's overview, the poster, when it was synced
257. let poster = syncPoster(show, details.poster_path)
258. res.requests = res.requests + poster.requests
259. res.posters = poster.saved
260. if (poster.failed != null) { res.errors = res.errors + 1 res.error = show.title + ': ' + poster.failed }
261. show.seasons = showSeasons
262. if (textOr(details.status) != null) { show.status = details.status }
263. if (details.number_of_seasons != null) { show.seasonsCount = details.number_of_seasons }
264. if (details.number_of_episodes != null) { show.episodesCount = details.number_of_episodes }
265. if (textOr(details.overview) != null) { show.tmdbSummary = details.overview }
266. show.image = poster.image
267. show.tmdbPoster = poster.tmdbPoster
268. show.tmdbSync = now()
269. if (showsTable.update(show.id, show) == null) { res.errors = res.errors + 1 res.error = show.title + ': show not updated: ' + showsTable.lastError() }
270. return res
271. }
273. // flush the three tables' index/meta state to disk (after a run, and every few shows)
274. static persistSync = () => {
275. showsTable.persist()
276. seasonsTable.persist()
277. episodesTable.persist()
278. return true
279. }
281. // totals of a run: the per-show counts added up
282. static emptyTotals = () => { return { shows = 0 requests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 errors = 0 } }
283. static addTotals = (t, r) => {
284. let o = t
285. o.shows = o.shows + 1
286. o.requests = o.requests + r.requests
287. o.newSeasons = o.newSeasons + r.newSeasons
288. o.newEpisodes = o.newEpisodes + r.newEpisodes
289. o.updatedEpisodes = o.updatedEpisodes + r.updatedEpisodes
290. o.updatedSeasons = o.updatedSeasons + r.updatedSeasons
291. o.posters = o.posters + r.posters
292. o.errors = o.errors + r.errors
293. return o
294. }
295. static totalsLine = (t) => {
296. return 'shows=' + t.shows + ' newSeasons=' + t.newSeasons + ' newEpisodes=' + t.newEpisodes + ' updatedEpisodes=' + t.updatedEpisodes
297. + ' updatedSeasons=' + t.updatedSeasons + ' posters=' + t.posters + ' requests=' + t.requests + ' errors=' + t.errors
298. }