tracker

29.7 KB

1. // tmdbsync.hl — THE TMDB SYNC (tracker.worldapi.org#9): for every show SOMEONE follows, fetch the show's
2. // details, seasons and episodes from TMDB and bring storage/mpackdb/ up to date, plus the show's poster.
3. // Used by BOTH runners: the daily run inside the app (project.hl `syncTick`, one show per step) and the
4. // command-line tool (tools/sync-tmdb.hl). Reference for the API use: the old tracker's
5. // crates/sync/commands/sync.js (read-only).
6. //
7. // GET <TMDB_BASE_URL>/tv/<tmdbId>?append_to_response=external_ids the details (status, counts,
8. // overview, poster_path, seasons[])
9. // + external_ids (imdb_id, tvdb_id)
10. // GET <TMDB_BASE_URL>/tv/<tmdbId>?append_to_response=season/1,season/2… up to 20 seasons with their
11. // episodes in ONE request
12. // GET <TMDB_IMAGE_URL>/w342<poster_path> the poster → storage/mpackdb/posters/
13. // Authorization: Bearer $TMDB_READ_TOKEN
14. // GET <TVMAZE_BASE_URL>/lookup/shows?imdb=tt… (else ?thetvdb=<id>) the TVmaze id, ONLY when the show
15. // has none (301 → /shows/<id>, 404 = unknown)
16. // GET <TVMAZE_BASE_URL>/shows/<tvmzId>/episodes a SERIES' episodes on TVmaze (#15), see below
17. // A MOVIE (`type = 'movie'`, 7814 of the migrated rows, 1 followed): only
18. // GET <TMDB_BASE_URL>/movie/<tmdbId>?append_to_response=external_ids its imdb id + poster; no seasons,
19. // status, overview (it is no TV show)
20. //
21. // EXTERNAL IDS (tracker.worldapi.org#12, the show page's link icons): a MISSING imdbId/tvdbId is filled from TMDB's
22. // external_ids, a missing tvmzId from TVmaze's lookup — an id the show already has is NEVER overwritten.
23. // RULES (ticket): new seasons/episodes are ADDED, existing ones UPDATED (title, summary, release) —
24. // matched by tmdbId, never duplicated; NOTHING is deleted. An episode without a tmdbId (161 migrated ones)
25. // is matched by its S/E number instead and gets the tmdbId. A season (no tmdbId in the migrated data) is
26. // matched by (show, seasonNumber). A TMDB value that is empty/null never overwrites a stored one.
27. // Season 0 ("Specials") is only synced for a show that already has a season 0 (else every followed show
28. // would suddenly flood /my/unwatched with specials); a TMDB season with no episodes is not created.
29. //
30. // TVMAZE MERGE (tracker.worldapi.org#15): AFTER TMDB, a series with a tvmzId gets TVmaze's episode list (one request)
31. // and TVmaze FILLS GAPS only — matched by season/episode number: a missing episode (and its season) is ADDED, an empty
32. // title or air date of an existing one is FILLED; a value TMDB gave is never overwritten. A placeholder title
33. // ("Episode 4", "TBA") counts as empty — and TMDB's own placeholder never overwrites a real title either (else the
34. // two would flip it every day). When TMDB later lists the episode, the TMDB step adopts it by number (it has no tmdbId)
35. // and TMDB's values win. TVmaze specials (no episode number) are skipped, season 0 only if the show has one. SAFETY:
36. // when the two number a show differently (fewer than 80 % of the episodes both know air within a day of each other or
37. // share the title — `numberingAgrees`), TVmaze is ignored for that show (`tvmazeSkipped`). Summaries are not taken
38. // (TVmaze's are HTML; the ticket asks for episodes, air dates, titles).
39. //
40. // IDS ARE SELF-ASSIGNED (randomBytes, like watches.hl): hl:mpackdb's own generated @id can repeat within
41. // one millisecond, which is what made update() lose rows (hybriel#113, fixed upstream, not in the vendored
42. // build) — a table holding only unique ids is safe to update().
43. //
44. // ONE PROCESS PER STORAGE: hl:mpackdb does not coordinate two processes (hybriel#21) — the in-app run uses
45. // the app's own open tables (imported from shows.hl); the tool only ever runs against a stopped app or a copy.
46. //
47. // Env: TMDB_READ_TOKEN (required unless TMDB_BASE_URL points at a fake), TMDB_BASE_URL
48. // (https://api.themoviedb.org/3), TMDB_IMAGE_URL (https://image.tmdb.org/t/p) — the gate points both at its
49. // fake TMDB (tests/browser.mjs).
50. import { fetch } from 'hl:fetch'
51. import { env } from 'hl:proc'
52. import { now } from 'hl:time'
53. import { randomBytes } from 'hl:crypto'
54. import { writeFile, mkDir, exists } from 'hl:fs'
55. import { showsTable, seasonsTable, episodesTable, storageDir } from './shows.hl'
57. static envOr = (name, fallback) => {
58. let v = env(name)
59. return v != null && v.trim() != '' ? v.trim() : fallback
60. }
61. static tmdbBase = envOr('TMDB_BASE_URL', 'https://api.themoviedb.org/3')
62. static tmdbImageBase = envOr('TMDB_IMAGE_URL', 'https://image.tmdb.org/t/p')
63. static tmdbToken = envOr('TMDB_READ_TOKEN', '')
64. static postersDir = storageDir + '/posters'
66. static tvmazeBase = envOr('TVMAZE_BASE_URL', 'https://api.tvmaze.com')
68. // the sync can run: a token is set (never printed) — the dev .env on Loreana has one, a gate sets its own
69. static syncEnabled = tmdbToken != ''
71. // TMDB allows 40 requests / 10 s: every runner waits this long per request it made before the next show
72. static msPerRequest = 260
73. // TVmaze allows 20 requests / 10 s (its API docs): 550 ms per lookup made
74. static msPerTvmazeRequest = 550
75. // how long a runner waits after a show before the next one (both runners: project.hl syncTick, tools/sync-tmdb.hl)
76. static pauseMsAfter = (r) => { return r.requests * msPerRequest + r.tvmazeRequests * msPerTvmazeRequest }
78. // one GET → the parsed JSON, or { failed = '<why>' }. A refused/timed-out fetch is an Error event (absorbed
79. // by the runner's `on Error`), the fetch then yields null.
80. static tmdbGet = (path) => {
81. let r = fetch(tmdbBase + path, { headers = { 'authorization' = 'Bearer ' + tmdbToken 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
82. if (r == null || r.status == null || r.status == 0) { return { failed = 'no answer for ' + path } }
83. if (r.status != 200) { return { failed = 'HTTP ' + r.status + ' for ' + path } }
84. let j = r.json()
85. if (j == null) { return { failed = 'no JSON for ' + path } }
86. return j
87. }
89. static textOr = (v) => { return v != null && hlTypeName(v) == 'String' && v.trim() != '' ? v : null }
91. // ---- external ids (tracker.worldapi.org#12) -----------------------------------------------------
92. static hasId = (v) => { return v != null && ('' + v).trim() != '' }
93. // an IMDb id as TMDB gives it ('tt' + digits), else null
94. static imdbOr = (v) => {
95. let t = textOr(v)
96. if (t == null || t.length < 3 || t.slice(0, 2) != 'tt') { return null }
97. let i = 2
98. while (i < t.length) {
99. if (!'0123456789'.includes(t[i])) { return null }
100. i = i + 1
101. }
102. return t
103. }
104. // a positive numeric id (TMDB's tvdb_id, TVmaze's id), else null
105. static numIdOr = (v) => { return v != null && hlTypeName(v) == 'Number' && v > 0 ? v : null }
107. // TVmaze's lookup by one external id: { id } (null = TVmaze doesn't know it, 404) or { failed }. hl:fetch follows the
108. // 301 to /shows/<id> by itself, so a hit answers the show's own JSON.
109. static tvmazeLookup = (param, value) => {
110. let r = fetch(tvmazeBase + '/lookup/shows?' + param + '=' + value, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
111. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for ' + param + '=' + value } }
112. if (r.status == 404) { return { id = null } }
113. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for ' + param + '=' + value } }
114. let j = r.json()
115. if (j == null || numIdOr(j.id) == null) { return { failed = 'TVmaze: no id for ' + param + '=' + value } }
116. return { id = j.id }
117. }
119. // the ids to SET on a show from TMDB's external_ids (+ TVmaze for a series): { imdbId, tvdbId, tvmzId } — each the
120. // show's own value when it has one (never overwritten), else the found one, else null — plus { filled, tvmazeRequests,
121. // failed }. TVmaze is asked by imdb, then by thetvdb, only while the show (after TMDB) still has no tvmzId.
122. static externalIdsFor = (show, ext, isSeries) => {
123. let out = { imdbId = show.imdbId tvdbId = show.tvdbId tvmzId = show.tvmzId filled = 0 tvmazeRequests = 0 failed = null }
124. let e = ext != null ? ext : {}
125. if (!hasId(out.imdbId) && imdbOr(e.imdb_id) != null) { out.imdbId = e.imdb_id out.filled = out.filled + 1 }
126. if (!isSeries) { return out }
127. if (!hasId(out.tvdbId) && numIdOr(e.tvdb_id) != null) { out.tvdbId = e.tvdb_id out.filled = out.filled + 1 }
128. if (hasId(out.tvmzId)) { return out }
129. let tries = []
130. if (imdbOr('' + out.imdbId) != null) { tries.push({ param = 'imdb' value = '' + out.imdbId }) }
131. if (numIdOr(out.tvdbId) != null) { tries.push({ param = 'thetvdb' value = '' + out.tvdbId }) }
132. for (t of tries) {
133. if (!hasId(out.tvmzId) && out.failed == null) {
134. out.tvmazeRequests = out.tvmazeRequests + 1
135. let found = tvmazeLookup(t.param, t.value)
136. if (found.failed != null) { out.failed = found.failed } else if (found.id != null) { out.tvmzId = found.id out.filled = out.filled + 1 }
137. }
138. }
139. return out
140. }
142. // ---- TVmaze merge (tracker.worldapi.org#15) ------------------------------------------------------
143. // a title that is none yet: empty, "TBA"/"TBD", or "Episode <n>" (what TMDB and TVmaze name an untitled episode; also
144. // French/German/Spanish: real data has TVmaze "Épisode 1" where TMDB says "Episode 1")
145. static placeholderWords = ['episode', 'épisode', 'Épisode', 'folge', 'episodio']
146. static isPlaceholderTitle = (v) => {
147. let t = textOr(v)
148. if (t == null) { return true }
149. let l = t.trim().toLowerCase()
150. if (l == 'tba' || l == 'tbd' || l == 'tba.' || l == 'to be announced') { return true }
151. let parts = l.split(' ')
152. if (parts.length != 2 || !placeholderWords.includes(parts[0]) || parts[1].length == 0) { return false }
153. let i = 0
154. while (i < parts[1].length) {
155. if (!'0123456789'.includes(parts[1][i])) { return false }
156. i = i + 1
157. }
158. return true
159. }
161. // 'YYYY-MM-DD' → a day number (only differences matter), null when it is no such date
162. static dayNumber = (v) => {
163. let t = textOr(v)
164. if (t == null || t.length < 10) { return null }
165. let y = toNumber(t.slice(0, 4))
166. let m = toNumber(t.slice(5, 7))
167. let d = toNumber(t.slice(8, 10))
168. if (y == null || m == null || d == null || m < 1 || m > 12) { return null }
169. let before = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]
170. let py = m <= 2 ? y - 1 : y
171. let leaps = (py - py % 4) / 4 - (py - py % 100) / 100 + (py - py % 400) / 400
172. return y * 365 + leaps + before[m - 1] + d
173. }
174. static daysApart = (a, b) => {
175. let x = dayNumber(a)
176. let y = dayNumber(b)
177. if (x == null || y == null) { return null }
178. return x > y ? x - y : y - x
179. }
181. // TVmaze's episode list of one show: a list, or { failed }
182. static tvmazeEpisodes = (tvmzId) => {
183. let r = fetch(tvmazeBase + '/shows/' + tvmzId + '/episodes', { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
184. if (r == null || r.status == null || r.status == 0) { return { failed = 'TVmaze: no answer for episodes of ' + tvmzId } }
185. if (r.status != 200) { return { failed = 'TVmaze: HTTP ' + r.status + ' for episodes of ' + tvmzId } }
186. let j = r.json()
187. if (j == null || j.length == null) { return { failed = 'TVmaze: no episode list for ' + tvmzId } }
188. return { list = j }
189. }
191. // do we and TVmaze number a show the same way? Of the episodes both know (same S/E), at least 80 % must air within a day
192. // of each other OR carry the same title (a streaming show TVmaze dates all on its release day, TMDB week by week, still
193. // agrees by title). A show we have episodes of but TVmaze numbers none of the same (e.g. seasons by year) never agrees.
194. // Answers { known, compared, agree, ok }.
195. static sameTitle = (a, b) => { return !isPlaceholderTitle(a) && !isPlaceholderTitle(b) && a.trim().toLowerCase() == b.trim().toLowerCase() }
196. static numberingAgrees = (epByNumber, list) => {
197. let out = { known = 0 compared = 0 agree = 0 ok = true }
198. let localCount = epByNumber.keys().length
199. for (te of list) {
200. let e = epByNumber[te.season + 'x' + te.number]
201. if (e != null) {
202. out.known = out.known + 1
203. let apart = daysApart(e.release, te.airdate)
204. let titled = !isPlaceholderTitle(e.title) && !isPlaceholderTitle(te.name)
205. if (apart != null || titled) {
206. out.compared = out.compared + 1
207. if ((apart != null && apart <= 1) || sameTitle(e.title, te.name)) { out.agree = out.agree + 1 }
208. }
209. }
210. }
211. out.ok = !((localCount > 0 && out.known == 0) || (out.compared > 0 && out.agree * 5 < out.compared * 4))
212. return out
213. }
215. // Merge TVmaze's episodes of a series into what we have AFTER the TMDB step (gaps only, rules at the top).
216. // `seasonsByNumber` / `epByNumber` = the show's seasons by number and episodes by '<s>x<e>', as the TMDB step left them.
217. // Answers { tvmazeRequests, newSeasons, newEpisodes, filledEpisodes, skipped, errors, error, showSeasons }.
218. static mergeTvmaze = (title, showKey, tvmzId, seasonsIn, episodesIn, showSeasonsIn) => {
219. let seasonsByNumber = seasonsIn
220. let epByNumber = episodesIn
221. let out = { tvmazeRequests = 1 newSeasons = 0 newEpisodes = 0 filledEpisodes = 0 skipped = 0 errors = 0 error = '' showSeasons = showSeasonsIn }
222. let got = tvmazeEpisodes(tvmzId)
223. if (got.failed != null) { out.errors = 1 out.error = title + ': ' + got.failed return out }
224. // the numbered episodes TVmaze has in seasons we take (no specials; season 0 only when the show has one)
225. let list = []
226. for (te of got.list) {
227. if (numIdOr(te.season) != null && numIdOr(te.number) != null) { list.push(te) }
228. else if (te.season == 0 && numIdOr(te.number) != null && seasonsByNumber['0'] != null) { list.push(te) }
229. }
230. if (!numberingAgrees(epByNumber, list).ok) { out.skipped = 1 return out }
232. let added = {}
233. let addedFirstDate = {}
234. for (te of list) {
235. let key = te.season + 'x' + te.number
236. let e = epByNumber[key]
237. if (e == null) {
238. let seasonKey = seasonsByNumber['' + te.season] != null ? seasonsByNumber['' + te.season].id : null
239. if (seasonKey == null) {
240. if (added['' + te.season] == null) { added['' + te.season] = [] addedFirstDate['' + te.season] = textOr(te.airdate) seasonsByNumber['' + te.season] = { id = newId() isNew = true } }
241. seasonKey = seasonsByNumber['' + te.season].id
242. }
243. let fresh = { id = newId() episodeNumber = te.number seasonNumber = te.season
244. title = textOr(te.name) summary = '' release = textOr(te.airdate)
245. show = showKey season = seasonKey tmdbId = null tvmzId = te.id oldId = 'tvmaze-episode:' + te.id }
246. if (episodesTable.put(fresh) == null) {
247. out.errors = out.errors + 1
248. out.error = title + ': TVmaze episode ' + te.id + ' not stored: ' + episodesTable.lastError()
249. } else {
250. if (added['' + te.season] == null) { added['' + te.season] = [] }
251. let l = added['' + te.season]
252. l.push(fresh.id)
253. added['' + te.season] = l
254. epByNumber[key] = fresh
255. out.newEpisodes = out.newEpisodes + 1
256. }
257. } else {
258. let changed = false
259. if (isPlaceholderTitle(e.title) && !isPlaceholderTitle(te.name)) { e.title = te.name changed = true }
260. if (textOr(e.release) == null && textOr(te.airdate) != null) { e.release = te.airdate changed = true }
261. if (changed) {
262. if (episodesTable.update(e.id, e) == null) {
263. out.errors = out.errors + 1
264. out.error = title + ': episode ' + e.id + ' not filled from TVmaze: ' + episodesTable.lastError()
265. } else {
266. epByNumber[key] = e
267. out.filledEpisodes = out.filledEpisodes + 1
268. }
269. }
270. }
271. }
272. // the seasons that got episodes: an existing one is updated, a new one stored and linked to the show
273. let showSeasons = []
274. for (sid of showSeasonsIn) { showSeasons.push(sid) }
275. for (n of added.keys()) {
276. let ids = added[n]
277. let s = seasonsByNumber[n]
278. if (ids.length > 0 && s.isNew == true) {
279. let season = { id = s.id seasonNumber = toNumber(n) title = 'Season ' + n release = addedFirstDate[n] summary = ''
280. show = showKey episodes = ids oldId = 'tvmaze-season:' + tvmzId + ':' + n }
281. if (seasonsTable.put(season) == null) {
282. out.errors = out.errors + 1
283. out.error = title + ': TVmaze season ' + n + ' not stored: ' + seasonsTable.lastError()
284. } else {
285. showSeasons.push(season.id)
286. out.newSeasons = out.newSeasons + 1
287. }
288. } else if (ids.length > 0) {
289. let season = seasonsTable.fetch(s.id)
290. let eps = []
291. for (eid of season.episodes) { eps.push(eid) }
292. for (eid of ids) { eps.push(eid) }
293. season.episodes = eps
294. if (seasonsTable.update(season.id, season) == null) {
295. out.errors = out.errors + 1
296. out.error = title + ': season ' + season.id + ' not updated: ' + seasonsTable.lastError()
297. }
298. }
299. }
300. out.showSeasons = showSeasons
301. return out
302. }
304. // ---- posters ----------------------------------------------------------------------------------
305. // the extension of a TMDB poster_path ('/abc.jpg' → 'jpg'); only the ones the poster route knows
306. static posterExt = (posterPath) => {
307. let parts = posterPath.split('.')
308. let ext = parts[parts.length - 1].toLowerCase()
309. if (ext == 'jpg' || ext == 'jpeg' || ext == 'png' || ext == 'webp') { return ext }
310. return null
311. }
313. // downloads the w342 poster when the file is missing or TMDB has a new one; answers { requests, saved, failed }
314. // and sets show.image / show.tmdbPoster on the (caller's) record — the file is <oldId>.<ext>, the name
315. // shows.hl posterName() already builds, served by project.hl's /posters/:name route
316. static syncPoster = (show, posterPath) => {
317. let out = { requests = 0 saved = 0 failed = null image = show.image tmdbPoster = show.tmdbPoster }
318. if (textOr(posterPath) == null || show.oldId == null) { return out }
319. let ext = posterExt(posterPath)
320. if (ext == null) { return out }
321. let file = postersDir + '/' + show.oldId + '.' + ext
322. if (exists(file) && show.tmdbPoster == posterPath && show.image == ext) { return out }
323. out.requests = 1
324. let r = fetch(tmdbImageBase + '/w342' + posterPath, { headers = { 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 20000 })
325. if (r == null || r.status != 200 || r.body == null || r.body.length == 0) {
326. out.failed = 'poster ' + posterPath + ': ' + (r == null ? 'no answer' : 'HTTP ' + r.status)
327. return out
328. }
329. mkDir(postersDir)
330. writeFile(file, toBytes(r.body), 420)
331. out.saved = 1
332. out.image = ext
333. out.tmdbPoster = posterPath
334. return out
335. }
337. // ---- one show -----------------------------------------------------------------------------------
338. static newId = () => { return randomBytes(8, 'hex') }
340. // which TMDB seasons to sync: every numbered one with episodes; 0 only when the show has a season 0 already
341. static wantedSeasonNumbers = (details, localByNumber) => {
342. let out = []
343. if (details.seasons == null) { return out }
344. for (s of details.seasons) {
345. let n = s.season_number
346. if (n != null && (n > 0 || localByNumber['' + n] != null) && (s.episode_count == null || s.episode_count > 0 || localByNumber['' + n] != null)) { out.push(n) }
347. }
348. return out
349. }
351. // the seasons in batches of 20 (TMDB's append_to_response limit)
352. static batchesOf = (list, size) => {
353. let out = []
354. let cur = []
355. for (x of list) {
356. cur.push(x)
357. if (cur.length == size) { out.push(cur) cur = [] }
358. }
359. if (cur.length > 0) { out.push(cur) }
360. return out
361. }
363. // fields of an episode/season from TMDB that differ from the stored record (null/empty never overwrites)
364. static changedText = (stored, fresh) => { return fresh != null && fresh != stored }
366. // a MOVIE: one request — its imdb id (if missing) and its poster; nothing else of the record is touched
367. static syncMovie = (show, res) => {
368. let out = res
369. out.requests = 1
370. let details = tmdbGet('/movie/' + show.tmdbId + '?append_to_response=external_ids')
371. if (details.failed != null) { out.errors = 1 out.error = show.title + ': ' + details.failed return out }
372. let ids = externalIdsFor(show, details.external_ids, false)
373. out.ids = ids.filled
374. let poster = syncPoster(show, details.poster_path)
375. out.requests = out.requests + poster.requests
376. out.posters = poster.saved
377. if (poster.failed != null) { out.errors = out.errors + 1 out.error = show.title + ': ' + poster.failed }
378. let m = show
379. m.imdbId = ids.imdbId
380. m.image = poster.image
381. m.tmdbPoster = poster.tmdbPoster
382. m.tmdbSync = now()
383. if (showsTable.update(m.id, m) == null) { out.errors = out.errors + 1 out.error = show.title + ': movie not updated: ' + showsTable.lastError() }
384. return out
385. }
387. // Sync ONE show (by our id). Answers counts: { requests, tvmazeRequests, newSeasons, newEpisodes, updatedEpisodes,
388. // updatedSeasons, posters, ids, tvmazeEpisodes, tvmazeSeasons, tvmazeFilled, tvmazeSkipped, errors, error }. Never throws for a TMDB failure — it is counted and the show left as is.
389. static syncShow = (showId) => {
390. let res = { requests = 0 tvmazeRequests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 ids = 0 tvmazeEpisodes = 0 tvmazeSeasons = 0 tvmazeFilled = 0 tvmazeSkipped = 0 errors = 0 error = '' }
391. let show = showsTable.fetch(showId)
392. if (show == null) { res.errors = 1 res.error = 'no show ' + showId return res }
393. if (show.tmdbId == null) { res.errors = 1 res.error = show.title + ': no tmdbId' return res }
394. if (show.type == 'movie') { return syncMovie(show, res) }
396. res.requests = 1
397. let details = tmdbGet('/tv/' + show.tmdbId + '?append_to_response=external_ids')
398. if (details.failed != null) { res.errors = 1 res.error = show.title + ': ' + details.failed return res }
400. // what we have: seasons by number, episodes by tmdbId and by S/E number
401. let seasonsByNumber = {}
402. let epByTmdb = {}
403. let epByNumber = {}
404. for (sid of show.seasons) {
405. let s = seasonsTable.fetch(sid)
406. if (s != null) {
407. seasonsByNumber['' + s.seasonNumber] = s
408. for (eid of s.episodes) {
409. let e = episodesTable.fetch(eid)
410. if (e != null) {
411. if (e.tmdbId != null) { epByTmdb['' + e.tmdbId] = e }
412. epByNumber[e.seasonNumber + 'x' + e.episodeNumber] = e
413. }
414. }
415. }
416. }
418. // plain locals for the record literals below: inside a literal, an entry named `show`/`season` would shadow
419. // the variable for every later entry (SPEC "A literal's entries are evaluated in order")
420. let showKey = show.id
421. let showTmdb = show.tmdbId
422. let showSeasons = []
423. for (sid of show.seasons) { showSeasons.push(sid) }
424. let showChanged = false
426. for (batch of batchesOf(wantedSeasonNumbers(details, seasonsByNumber), 20)) {
427. let names = []
428. for (n of batch) { names.push('season/' + n) }
429. res.requests = res.requests + 1
430. let withSeasons = tmdbGet('/tv/' + show.tmdbId + '?append_to_response=' + names.join(','))
431. if (withSeasons.failed != null) {
432. res.errors = res.errors + 1
433. res.error = show.title + ': ' + withSeasons.failed
434. } else {
435. for (n of batch) {
436. let ts = withSeasons['season/' + n]
437. if (ts != null && ts.episodes != null) {
438. let season = seasonsByNumber['' + n]
439. let seasonIsNew = season == null
440. let seasonChanged = false
441. if (seasonIsNew) {
442. season = { id = newId() seasonNumber = n title = textOr(ts.name) != null ? ts.name : 'Season ' + n
443. release = textOr(ts.air_date) summary = textOr(ts.overview) != null ? ts.overview : ''
444. show = showKey episodes = [] oldId = 'tmdb-season:' + showTmdb + ':' + n }
445. } else {
446. if (changedText(season.title, textOr(ts.name))) { season.title = ts.name seasonChanged = true }
447. if (changedText(season.summary, textOr(ts.overview))) { season.summary = ts.overview seasonChanged = true }
448. if (changedText(season.release, textOr(ts.air_date))) { season.release = ts.air_date seasonChanged = true }
449. }
450. let seasonKey = season.id
451. let seasonEpisodes = []
452. for (eid of season.episodes) { seasonEpisodes.push(eid) }
453. for (te of ts.episodes) {
454. if (te.id != null && te.episode_number != null) {
455. let e = epByTmdb['' + te.id]
456. if (e == null) {
457. let byNum = epByNumber[n + 'x' + te.episode_number]
458. if (byNum != null && byNum.tmdbId == null) { e = byNum }
459. }
460. if (e == null) {
461. let fresh = { id = newId() episodeNumber = te.episode_number seasonNumber = n
462. title = textOr(te.name) summary = textOr(te.overview) != null ? te.overview : '' release = textOr(te.air_date)
463. show = showKey season = seasonKey tmdbId = te.id oldId = 'tmdb-episode:' + te.id }
464. let putId = episodesTable.put(fresh)
465. if (putId == null) {
466. res.errors = res.errors + 1
467. res.error = show.title + ': episode ' + te.id + ' not stored: ' + episodesTable.lastError()
468. } else {
469. seasonEpisodes.push(fresh.id)
470. epByTmdb['' + te.id] = fresh
471. epByNumber[n + 'x' + te.episode_number] = fresh
472. res.newEpisodes = res.newEpisodes + 1
473. seasonChanged = true
474. }
475. } else {
476. let changed = false
477. if (e.tmdbId == null) { e.tmdbId = te.id changed = true }
478. // #15: TMDB's "Episode 4" never replaces a real title (one TVmaze filled, or the migrated one)
479. if (changedText(e.title, textOr(te.name)) && !(isPlaceholderTitle(te.name) && !isPlaceholderTitle(e.title))) { e.title = te.name changed = true }
480. if (changedText(e.summary, textOr(te.overview))) { e.summary = te.overview changed = true }
481. if (changedText(e.release, textOr(te.air_date))) { e.release = te.air_date changed = true }
482. if (changed) {
483. if (episodesTable.update(e.id, e) == null) {
484. res.errors = res.errors + 1
485. res.error = show.title + ': episode ' + e.id + ' not updated: ' + episodesTable.lastError()
486. } else {
487. epByTmdb['' + e.tmdbId] = e
488. epByNumber[e.seasonNumber + 'x' + e.episodeNumber] = e
489. res.updatedEpisodes = res.updatedEpisodes + 1
490. }
491. }
492. }
493. }
494. }
495. season.episodes = seasonEpisodes
496. if (seasonIsNew) {
497. if (seasonsTable.put(season) == null) {
498. res.errors = res.errors + 1
499. res.error = show.title + ': season ' + n + ' not stored: ' + seasonsTable.lastError()
500. } else {
501. seasonsByNumber['' + n] = season
502. showSeasons.push(season.id)
503. showChanged = true
504. res.newSeasons = res.newSeasons + 1
505. }
506. } else if (seasonChanged) {
507. if (seasonsTable.update(season.id, season) == null) {
508. res.errors = res.errors + 1
509. res.error = show.title + ': season ' + season.id + ' not updated: ' + seasonsTable.lastError()
510. } else {
511. res.updatedSeasons = res.updatedSeasons + 1
512. }
513. }
514. }
515. }
516. }
517. }
519. // the external ids (tracker.worldapi.org#12): missing ones from TMDB's external_ids, a missing TVmaze id from TVmaze
520. let ids = externalIdsFor(show, details.external_ids, true)
521. res.ids = ids.filled
522. res.tvmazeRequests = ids.tvmazeRequests
523. if (ids.failed != null) { res.errors = res.errors + 1 res.error = show.title + ': ' + ids.failed }
525. // tracker.worldapi.org#15: what TVmaze has and TMDB not yet (new episodes/seasons, empty titles and air dates)
526. if (numIdOr(ids.tvmzId) != null) {
527. let tvm = mergeTvmaze(show.title, showKey, ids.tvmzId, seasonsByNumber, epByNumber, showSeasons)
528. res.tvmazeRequests = res.tvmazeRequests + tvm.tvmazeRequests
529. res.tvmazeEpisodes = tvm.newEpisodes
530. res.tvmazeSeasons = tvm.newSeasons
531. res.tvmazeFilled = tvm.filledEpisodes
532. res.tvmazeSkipped = tvm.skipped
533. if (tvm.errors > 0) { res.errors = res.errors + tvm.errors res.error = tvm.error }
534. showSeasons = tvm.showSeasons
535. }
537. // the show itself: status, counts, TMDB's overview, the poster, when it was synced
538. let poster = syncPoster(show, details.poster_path)
539. res.requests = res.requests + poster.requests
540. res.posters = poster.saved
541. if (poster.failed != null) { res.errors = res.errors + 1 res.error = show.title + ': ' + poster.failed }
542. show.seasons = showSeasons
543. if (textOr(details.status) != null) { show.status = details.status }
544. if (details.number_of_seasons != null) { show.seasonsCount = details.number_of_seasons }
545. if (details.number_of_episodes != null) { show.episodesCount = details.number_of_episodes }
546. if (textOr(details.overview) != null) { show.tmdbSummary = details.overview }
547. show.image = poster.image
548. show.tmdbPoster = poster.tmdbPoster
549. show.imdbId = ids.imdbId
550. show.tvdbId = ids.tvdbId
551. show.tvmzId = ids.tvmzId
552. show.tmdbSync = now()
553. if (showsTable.update(show.id, show) == null) { res.errors = res.errors + 1 res.error = show.title + ': show not updated: ' + showsTable.lastError() }
554. return res
555. }
557. // flush the three tables' index/meta state to disk (after a run, and every few shows)
558. static persistSync = () => {
559. showsTable.persist()
560. seasonsTable.persist()
561. episodesTable.persist()
562. return true
563. }
565. // totals of a run: the per-show counts added up
566. static emptyTotals = () => { return { shows = 0 requests = 0 tvmazeRequests = 0 newSeasons = 0 newEpisodes = 0 updatedEpisodes = 0 updatedSeasons = 0 posters = 0 ids = 0 tvmazeEpisodes = 0 tvmazeSeasons = 0 tvmazeFilled = 0 tvmazeSkipped = 0 errors = 0 } }
567. static addTotals = (t, r) => {
568. let o = t
569. o.shows = o.shows + 1
570. o.requests = o.requests + r.requests
571. o.tvmazeRequests = o.tvmazeRequests + r.tvmazeRequests
572. o.ids = o.ids + r.ids
573. o.newSeasons = o.newSeasons + r.newSeasons
574. o.newEpisodes = o.newEpisodes + r.newEpisodes
575. o.updatedEpisodes = o.updatedEpisodes + r.updatedEpisodes
576. o.updatedSeasons = o.updatedSeasons + r.updatedSeasons
577. o.posters = o.posters + r.posters
578. o.tvmazeEpisodes = o.tvmazeEpisodes + r.tvmazeEpisodes
579. o.tvmazeSeasons = o.tvmazeSeasons + r.tvmazeSeasons
580. o.tvmazeFilled = o.tvmazeFilled + r.tvmazeFilled
581. o.tvmazeSkipped = o.tvmazeSkipped + r.tvmazeSkipped
582. o.errors = o.errors + r.errors
583. return o
584. }
585. static totalsLine = (t) => {
586. return 'shows=' + t.shows + ' newSeasons=' + t.newSeasons + ' newEpisodes=' + t.newEpisodes + ' updatedEpisodes=' + t.updatedEpisodes
587. + ' updatedSeasons=' + t.updatedSeasons + ' posters=' + t.posters + ' ids=' + t.ids + ' requests=' + t.requests + ' tvmazeRequests=' + t.tvmazeRequests + ' errors=' + t.errors
588. + ' tvmazeEpisodes=' + t.tvmazeEpisodes + ' tvmazeSeasons=' + t.tvmazeSeasons + ' tvmazeFilled=' + t.tvmazeFilled + ' tvmazeSkipped=' + t.tvmazeSkipped
589. }