tracker

19.7 KB

1. // tmdbsync.hl — THE TMDB SYNC (tracker.worldapi.org#9): for every show SOMEONE follows, fetch the show's
2. // details, seasons and episodes from TMDB and bring storage/mpackdb/ up to date, plus the show's poster.
3. // Used by BOTH runners: the daily run inside the app (project.hl `syncTick`, one show per step) and the
4. // command-line tool (tools/sync-tmdb.hl). Reference for the API use: the old tracker's
5. // crates/sync/commands/sync.js (read-only).
6. //
7. // GET <TMDB_BASE_URL>/tv/<tmdbId>?append_to_response=external_ids the details (status, counts,
8. // overview, poster_path, seasons[])
9. // + external_ids (imdb_id, tvdb_id)
10. // GET <TMDB_BASE_URL>/tv/<tmdbId>?append_to_response=season/1,season/2… up to 20 seasons with their
11. // episodes in ONE request
12. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster → storage/mpackdb/posters/
13. // Authorization: Bearer $TMDB_READ_TOKEN
14. // GET <TVMAZE_BASE_URL>/lookup/shows?imdb=tt… (else ?thetvdb=<id>) the TVmaze id, ONLY when the show
15. // has none (301 → /shows/<id>, 404 = unknown)
16. // A MOVIE (`type = 'movie'`, 7814 of the migrated rows, 1 followed): only
17. // GET <TMDB_BASE_URL>/movie/<tmdbId>?append_to_response=external_ids its imdb id + poster; no seasons,
18. // status, overview (it is no TV show)
19. //
20. // EXTERNAL IDS (tracker.worldapi.org#12, the show page's link icons): a MISSING imdbId/tvdbId is filled from TMDB's
21. // external_ids, a missing tvmzId from TVmaze's lookup — an id the show already has is NEVER overwritten.
22. // RULES (ticket): new seasons/episodes are ADDED, existing ones UPDATED (title, summary, release) —
23. // matched by tmdbId, never duplicated; NOTHING is deleted. An episode without a tmdbId (161 migrated ones)
24. // is matched by its S/E number instead and gets the tmdbId. A season (no tmdbId in the migrated data) is
25. // matched by (show, seasonNumber). A TMDB value that is empty/null never overwrites a stored one.
26. // Season 0 ("Specials") is only synced for a show that already has a season 0 (else every followed show
27. // would suddenly flood /my/unwatched with specials); a TMDB season with no episodes is not created.
28. //
29. // IDS ARE SELF-ASSIGNED (randomBytes, like watches.hl): hl:mpackdb's own generated @id can repeat within
30. // one millisecond, which is what made update() lose rows (hybriel#113, fixed upstream, not in the vendored
31. // build) — a table holding only unique ids is safe to update().
32. //
33. // ONE PROCESS PER STORAGE: hl:mpackdb does not coordinate two processes (hybriel#21) — the in-app run uses
34. // the app's own open tables (imported from shows.hl); the tool only ever runs against a stopped app or a copy.
35. //
36. // Env: TMDB_READ_TOKEN (required unless TMDB_BASE_URL points at a fake), TMDB_BASE_URL
37. // (https://api.themoviedb.org/3), TMDB_IMAGE_URL (https://image.tmdb.org/t/p) — the gate points both at its
38. // fake TMDB (tests/browser.mjs).
39. import { fetch } from 'hl:fetch'
40. import { env } from 'hl:proc'
41. import { now } from 'hl:time'
42. import { randomBytes } from 'hl:crypto'
43. import { writeFile, mkDir, exists } from 'hl:fs'
44. import { showsTable, seasonsTable, episodesTable, storageDir } from './shows.hl'
46. static envOr = (name, fallback) => {
47. let v = env(name)
48. return v != null && v.trim() != '' ? v.trim() : fallback
49. }
50. static tmdbBase = envOr('TMDB_BASE_URL', 'https://api.themoviedb.org/3')
51. static tmdbImageBase = envOr('TMDB_IMAGE_URL', 'https://image.tmdb.org/t/p')
52. static tmdbToken = envOr('TMDB_READ_TOKEN', '')
53. static postersDir = storageDir + '/posters'
55. static tvmazeBase = envOr('TVMAZE_BASE_URL', 'https://api.tvmaze.com')
57. // the sync can run: a token is set (never printed) — the dev .env on Loreana has one, a gate sets its own
58. static syncEnabled = tmdbToken != ''
60. // TMDB allows 40 requests / 10 s: every runner waits this long per request it made before the next show
61. static msPerRequest = 260
62. // TVmaze allows 20 requests / 10 s (its API docs): 550 ms per lookup made
63. static msPerTvmazeRequest = 550
64. // how long a runner waits after a show before the next one (both runners: project.hl syncTick, tools/sync-tmdb.hl)
65. static pauseMsAfter = (r) => { return r.requests * msPerRequest + r.tvmazeRequests * msPerTvmazeRequest }
67. // one GET → the parsed JSON, or { failed = '<why>' }. A refused/timed-out fetch is an Error event (absorbed
68. // by the runner's `on Error`), the fetch then yields null.
69. static tmdbGet = (path) => {
70. let r = fetch(tmdbBase + path, { headers = { 'authorization' = 'Bearer ' + tmdbToken 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
71. if (r == null || r.status == null || r.status == 0) { return { failed = 'no answer for ' + path } }
72. if (r.status != 200) { return { failed = 'HTTP ' + r.status + ' for ' + path } }
73. let j = r.json()
74. if (j == null) { return { failed = 'no JSON for ' + path } }
75. return j
76. }
78. static textOr = (v) => { return v != null && hlTypeName(v) == 'String' && v.trim() != '' ? v : null }
80. // ---- external ids (tracker.worldapi.org#12) -----------------------------------------------------
81. static hasId = (v) => { return v != null && ('' + v).trim() != '' }
82. // an IMDb id as TMDB gives it ('tt' + digits), else null
83. static imdbOr = (v) => {
84. let t = textOr(v)
85. if (t == null || t.length < 3 || t.slice(0, 2) != 'tt') { return null }
86. let i = 2
87. while (i < t.length) {
88. if (!'0123456789'.includes(t[i])) { return null }
89. i = i + 1
90. }
91. return t
92. }
93. // a positive numeric id (TMDB's tvdb_id, TVmaze's id), else null
94. static numIdOr = (v) => { return v != null && hlTypeName(v) == 'Number' && v > 0 ? v : null }
96. // TVmaze's lookup by one external id: { id } (null = TVmaze doesn't know it, 404) or { failed }. hl:fetch follows the
97. // 301 to /shows/<id> by itself, so a hit answers the show's own JSON.
98. static tvmazeLookup = (param, value) => {
99. let r = fetch(tvmazeBase + '/lookup/shows?' + param + '=' + value, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
100. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for ' + param + '=' + value } }
101. if (r.status == 404) { return { id = null } }
102. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for ' + param + '=' + value } }
103. let j = r.json()
104. if (j == null || numIdOr(j.id) == null) { return { failed = 'TVmaze: no id for ' + param + '=' + value } }
105. return { id = j.id }
106. }
108. // the ids to SET on a show from TMDB's external_ids (+ TVmaze for a series): { imdbId, tvdbId, tvmzId } — each the
109. // show's own value when it has one (never overwritten), else the found one, else null — plus { filled, tvmazeRequests,
110. // failed }. TVmaze is asked by imdb, then by thetvdb, only while the show (after TMDB) still has no tvmzId.
111. static externalIdsFor = (show, ext, isSeries) => {
112. let out = { imdbId = show.imdbId tvdbId = show.tvdbId tvmzId = show.tvmzId filled = 0 tvmazeRequests = 0 failed = null }
113. let e = ext != null ? ext : {}
114. if (!hasId(out.imdbId) && imdbOr(e.imdb_id) != null) { out.imdbId = e.imdb_id out.filled = out.filled + 1 }
115. if (!isSeries) { return out }
116. if (!hasId(out.tvdbId) && numIdOr(e.tvdb_id) != null) { out.tvdbId = e.tvdb_id out.filled = out.filled + 1 }
117. if (hasId(out.tvmzId)) { return out }
118. let tries = []
119. if (imdbOr('' + out.imdbId) != null) { tries.push({ param = 'imdb' value = '' + out.imdbId }) }
120. if (numIdOr(out.tvdbId) != null) { tries.push({ param = 'thetvdb' value = '' + out.tvdbId }) }
121. for (t of tries) {
122. if (!hasId(out.tvmzId) && out.failed == null) {
123. out.tvmazeRequests = out.tvmazeRequests + 1
124. let found = tvmazeLookup(t.param, t.value)
125. if (found.failed != null) { out.failed = found.failed } else if (found.id != null) { out.tvmzId = found.id out.filled = out.filled + 1 }
126. }
127. }
128. return out
129. }
131. // ---- posters ----------------------------------------------------------------------------------
132. // the extension of a TMDB poster_path ('/abc.jpg' → 'jpg'); only the ones the poster route knows
133. static posterExt = (posterPath) => {
134. let parts = posterPath.split('.')
135. let ext = parts[parts.length - 1].toLowerCase()
136. if (ext == 'jpg' || ext == 'jpeg' || ext == 'png' || ext == 'webp') { return ext }
137. return null
138. }
140. // downloads the w342 poster when the file is missing or TMDB has a new one; answers { requests, saved, failed }
141. // and sets show.image / show.tmdbPoster on the (caller's) record — the file is <oldId>.<ext>, the name
142. // shows.hl posterName() already builds, served by project.hl's /posters/:name route
143. static syncPoster = (show, posterPath) => {
144. let out = { requests = 0 saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
145. if (textOr(posterPath) == null || show.oldId == null) { return out }
146. let ext = posterExt(posterPath)
147. if (ext == null) { return out }
148. let file = postersDir + '/' + show.oldId + '.' + ext
149. if (exists(file) && show.tmdbPoster == posterPath && show.image == ext) { return out }
150. out.requests = 1
151. let r = fetch(tmdbImageBase + '/w342' + posterPath, { headers = { 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
152. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
153. out.failed = 'poster ' + posterPath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
154. return out
155. }
156. mkDir(postersDir)
157. writeFile(file, toBytes(r.body), 420)
158. out.saved = 1
159. out.image = ext
160. out.tmdbPoster = posterPath
161. return out
162. }
164. // ---- one show -----------------------------------------------------------------------------------
165. static newId = () => { return randomBytes(8, 'hex') }
167. // which TMDB seasons to sync: every numbered one with episodes; 0 only when the show has a season 0 already
168. static wantedSeasonNumbers = (details, localByNumber) => {
169. let out = []
170. if (details.seasons == null) { return out }
171. for (s of details.seasons) {
172. let n = s.season_number
173. if (n != null && (n > 0 || localByNumber['' + n] != null) && (s.episode_count == null || s.episode_count > 0 || localByNumber['' + n] != null)) { out.push(n) }
174. }
175. return out
176. }
178. // the seasons in batches of 20 (TMDB's append_to_response limit)
179. static batchesOf = (list, size) => {
180. let out = []
181. let cur = []
182. for (x of list) {
183. cur.push(x)
184. if (cur.length == size) { out.push(cur) cur = [] }
185. }
186. if (cur.length > 0) { out.push(cur) }
187. return out
188. }
190. // fields of an episode/season from TMDB that differ from the stored record (null/empty never overwrites)
191. static changedText = (stored, fresh) => { return fresh != null && fresh != stored }
193. // a MOVIE: one request — its imdb id (if missing) and its poster; nothing else of the record is touched
194. static syncMovie = (show, res) => {
195. let out = res
196. out.requests = 1
197. let details = tmdbGet('/movie/' + show.tmdbId + '?append_to_response=external_ids')
198. if (details.failed != null) { out.errors = 1 out.error = show.title + ': ' + details.failed return out }
199. let ids = externalIdsFor(show, details.external_ids, false)
200. out.ids = ids.filled
201. let poster = syncPoster(show, details.poster_path)
202. out.requests = out.requests + poster.requests
203. out.posters = poster.saved
204. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
205. let m = show
206. m.imdbId = ids.imdbId
207. m.image = poster.image
208. m.tmdbPoster = poster.tmdbPoster
209. m.tmdbSync = now()
210. if (showsTable.update(m.id, m) == null) { out.errors = out.errors + 1 out.error = show.title + ': movie not updated: ' + showsTable.lastError() }
211. return out
212. }
214. // Sync ONE show (by our id). Answers counts: { requests, tvmazeRequests, newSeasons, newEpisodes, updatedEpisodes,
215. // updatedSeasons, posters, ids, errors, error }. Never throws for a TMDB failure — it is counted and the show left as is.
216. static syncShow = (showId) => {
217. let res = { requests = 0 tvmazeRequests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 ids = 0 errors = 0 error = '' }
218. let show = showsTable.fetch(showId)
219. if (show == null) { res.errors = 1 res.error = 'no show ' + showId return res }
220. if (show.tmdbId == null) { res.errors = 1 res.error = show.title + ': no tmdbId' return res }
221. if (show.type == 'movie') { return syncMovie(show, res) }
223. res.requests = 1
224. let details = tmdbGet('/tv/' + show.tmdbId + '?append_to_response=external_ids')
225. if (details.failed != null) { res.errors = 1 res.error = show.title + ': ' + details.failed return res }
227. // what we have: seasons by number, episodes by tmdbId and by S/E number
228. let seasonsByNumber = {}
229. let epByTmdb = {}
230. let epByNumber = {}
231. for (sid of show.seasons) {
232. let s = seasonsTable.fetch(sid)
233. if (s != null) {
234. seasonsByNumber['' + s.seasonNumber] = s
235. for (eid of s.episodes) {
236. let e = episodesTable.fetch(eid)
237. if (e != null) {
238. if (e.tmdbId != null) { epByTmdb['' + e.tmdbId] = e }
239. epByNumber[e.seasonNumber + 'x' + e.episodeNumber] = e
240. }
241. }
242. }
243. }
245. // plain locals for the record literals below: inside a literal, an entry named `show`/`season` would shadow
246. // the variable for every later entry (SPEC "A literal's entries are evaluated in order")
247. let showKey = show.id
248. let showTmdb = show.tmdbId
249. let showSeasons = []
250. for (sid of show.seasons) { showSeasons.push(sid) }
251. let showChanged = false
253. for (batch of batchesOf(wantedSeasonNumbers(details, seasonsByNumber), 20)) {
254. let names = []
255. for (n of batch) { names.push('season/' + n) }
256. res.requests = res.requests + 1
257. let withSeasons = tmdbGet('/tv/' + show.tmdbId + '?append_to_response=' + names.join(','))
258. if (withSeasons.failed != null) {
259. res.errors = res.errors + 1
260. res.error = show.title + ': ' + withSeasons.failed
261. } else {
262. for (n of batch) {
263. let ts = withSeasons['season/' + n]
264. if (ts != null && ts.episodes != null) {
265. let season = seasonsByNumber['' + n]
266. let seasonIsNew = season == null
267. let seasonChanged = false
268. if (seasonIsNew) {
269. season = { id = newId() seasonNumber = n title = textOr(ts.name) != null ? ts.name : 'Season ' + n
270. release = textOr(ts.air_date) summary = textOr(ts.overview) != null ? ts.overview : ''
271. show = showKey episodes = [] oldId = 'tmdb-season:' + showTmdb + ':' + n }
272. } else {
273. if (changedText(season.title, textOr(ts.name))) { season.title = ts.name seasonChanged = true }
274. if (changedText(season.summary, textOr(ts.overview))) { season.summary = ts.overview seasonChanged = true }
275. if (changedText(season.release, textOr(ts.air_date))) { season.release = ts.air_date seasonChanged = true }
276. }
277. let seasonKey = season.id
278. let seasonEpisodes = []
279. for (eid of season.episodes) { seasonEpisodes.push(eid) }
280. for (te of ts.episodes) {
281. if (te.id != null && te.episode_number != null) {
282. let e = epByTmdb['' + te.id]
283. if (e == null) {
284. let byNum = epByNumber[n + 'x' + te.episode_number]
285. if (byNum != null && byNum.tmdbId == null) { e = byNum }
286. }
287. if (e == null) {
288. let fresh = { id = newId() episodeNumber = te.episode_number seasonNumber = n
289. title = textOr(te.name) summary = textOr(te.overview) != null ? te.overview : '' release = textOr(te.air_date)
290. show = showKey season = seasonKey tmdbId = te.id oldId = 'tmdb-episode:' + te.id }
291. let putId = episodesTable.put(fresh)
292. if (putId == null) {
293. res.errors = res.errors + 1
294. res.error = show.title + ': episode ' + te.id + ' not stored: ' + episodesTable.lastError()
295. } else {
296. seasonEpisodes.push(fresh.id)
297. epByTmdb['' + te.id] = fresh
298. res.newEpisodes = res.newEpisodes + 1
299. seasonChanged = true
300. }
301. } else {
302. let changed = false
303. if (e.tmdbId == null) { e.tmdbId = te.id changed = true }
304. if (changedText(e.title, textOr(te.name))) { e.title = te.name changed = true }
305. if (changedText(e.summary, textOr(te.overview))) { e.summary = te.overview changed = true }
306. if (changedText(e.release, textOr(te.air_date))) { e.release = te.air_date changed = true }
307. if (changed) {
308. if (episodesTable.update(e.id, e) == null) {
309. res.errors = res.errors + 1
310. res.error = show.title + ': episode ' + e.id + ' not updated: ' + episodesTable.lastError()
311. } else {
312. epByTmdb['' + e.tmdbId] = e
313. res.updatedEpisodes = res.updatedEpisodes + 1
314. }
315. }
316. }
317. }
318. }
319. season.episodes = seasonEpisodes
320. if (seasonIsNew) {
321. if (seasonsTable.put(season) == null) {
322. res.errors = res.errors + 1
323. res.error = show.title + ': season ' + n + ' not stored: ' + seasonsTable.lastError()
324. } else {
325. seasonsByNumber['' + n] = season
326. showSeasons.push(season.id)
327. showChanged = true
328. res.newSeasons = res.newSeasons + 1
329. }
330. } else if (seasonChanged) {
331. if (seasonsTable.update(season.id, season) == null) {
332. res.errors = res.errors + 1
333. res.error = show.title + ': season ' + season.id + ' not updated: ' + seasonsTable.lastError()
334. } else {
335. res.updatedSeasons = res.updatedSeasons + 1
336. }
337. }
338. }
339. }
340. }
341. }
343. // the external ids (tracker.worldapi.org#12): missing ones from TMDB's external_ids, a missing TVmaze id from TVmaze
344. let ids = externalIdsFor(show, details.external_ids, true)
345. res.ids = ids.filled
346. res.tvmazeRequests = ids.tvmazeRequests
347. if (ids.failed != null) { res.errors = res.errors + 1 res.error = show.title + ': ' + ids.failed }
349. // the show itself: status, counts, TMDB's overview, the poster, when it was synced
350. let poster = syncPoster(show, details.poster_path)
351. res.requests = res.requests + poster.requests
352. res.posters = poster.saved
353. if (poster.failed != null) { res.errors = res.errors + 1 res.error = show.title + ': ' + poster.failed }
354. show.seasons = showSeasons
355. if (textOr(details.status) != null) { show.status = details.status }
356. if (details.number_of_seasons != null) { show.seasonsCount = details.number_of_seasons }
357. if (details.number_of_episodes != null) { show.episodesCount = details.number_of_episodes }
358. if (textOr(details.overview) != null) { show.tmdbSummary = details.overview }
359. show.image = poster.image
360. show.tmdbPoster = poster.tmdbPoster
361. show.imdbId = ids.imdbId
362. show.tvdbId = ids.tvdbId
363. show.tvmzId = ids.tvmzId
364. show.tmdbSync = now()
365. if (showsTable.update(show.id, show) == null) { res.errors = res.errors + 1 res.error = show.title + ': show not updated: ' + showsTable.lastError() }
366. return res
367. }
369. // flush the three tables' index/meta state to disk (after a run, and every few shows)
370. static persistSync = () => {
371. showsTable.persist()
372. seasonsTable.persist()
373. episodesTable.persist()
374. return true
375. }
377. // totals of a run: the per-show counts added up
378. static emptyTotals = () => { return { shows = 0 requests = 0 tvmazeRequests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 ids = 0 errors = 0 } }
379. static addTotals = (t, r) => {
380. let o = t
381. o.shows = o.shows + 1
382. o.requests = o.requests + r.requests
383. o.tvmazeRequests = o.tvmazeRequests + r.tvmazeRequests
384. o.ids = o.ids + r.ids
385. o.newSeasons = o.newSeasons + r.newSeasons
386. o.newEpisodes = o.newEpisodes + r.newEpisodes
387. o.updatedEpisodes = o.updatedEpisodes + r.updatedEpisodes
388. o.updatedSeasons = o.updatedSeasons + r.updatedSeasons
389. o.posters = o.posters + r.posters
390. o.errors = o.errors + r.errors
391. return o
392. }
393. static totalsLine = (t) => {
394. return 'shows=' + t.shows + ' newSeasons=' + t.newSeasons + ' newEpisodes=' + t.newEpisodes + ' updatedEpisodes=' + t.updatedEpisodes
395. + ' updatedSeasons=' + t.updatedSeasons + ' posters=' + t.posters + ' ids=' + t.ids + ' requests=' + t.requests + ' tvmazeRequests=' + t.tvmazeRequests + ' errors=' + t.errors
396. }