tracker

5.2 KB

1. // tools/sync-tmdb.hl — THE TMDB SYNC FROM THE COMMAND LINE (tracker.worldapi.org#9): the same per-show sync
2. // the app runs every day (tmdbsync.hl), for every followed show, one after the other, waiting between shows
3. // so TMDB's 40 requests / 10 s (and TVmaze's 20 / 10 s) are never exceeded. Idempotent: a second run adds nothing (prints 0 new).
4. //
5. // STOP THE APP FIRST (or run it on a COPY): hl:mpackdb does not coordinate two processes on one storage
6. // directory (hybriel#21). The running app does this itself every day (project.hl `syncTick`) — this tool is
7. // for tests, the real-data proof and a manual catch-up.
8. //
9. // set -a; . ./.env; set +a # TMDB_READ_TOKEN — never print it
10. // TRACKER_STORAGE=/abs/path/storage/mpackdb ./bin/hybriel tools/sync-tmdb.hl [--limit N] [--delta [--since YYYY-MM-DD] [--plan]]
11. //
12. // Default: the FULL WALK (every followed show). --delta (tracker#18): what the app's daily run does — the change lists since
13. // the last run (deltasync.hl; `--since` overrides the stored day), then only the changed titles of ours; stores the day at the
14. // end like the app (not with --since or --limit). --plan: only the change lists and the queue they give, no title synced.
15. //
16. // Prints one line per show and the totals: shows, newSeasons, newEpisodes, updatedEpisodes, updatedSeasons,
17. // posters, ids (external ids filled, tracker.worldapi.org#12), requests, tvmazeRequests, errors, seconds, and (#15) what
18. // TVmaze added: tvmazeEpisodes, tvmazeSeasons, tvmazeFilled (episodes whose empty title/air date it filled), tvmazeSkipped
19. // (shows TVmaze numbers differently, left alone).
20. import { env, args, exit } from 'hl:proc'
21. import { now, sleep, timestamp } from 'hl:time'
22. import { syncEnabled, syncShow, syncShowDelta, persistSync, emptyTotals, addTotals, totalsLine, pauseMsAfter } from '../tmdbsync.hl'
23. import { allFollowedShowIds } from '../follows.hl'
24. import { applySynced, persistDetails } from '../details.hl'
25. import { planStart, planStep, planIds, planLine, markSynced, isFollowedInRun } from '../deltasync.hl'
27. storageDir = env('TRACKER_STORAGE')
28. if (storageDir == null || !storageDir.startsWith('/')) { console.log('REFUSED: set TRACKER_STORAGE, absolute (app stopped, or a copy)') exit(2) }
29. if (!syncEnabled) { console.log('REFUSED: TMDB_READ_TOKEN is not set') exit(2) }
31. limit = -1
32. delta = false
33. planOnly = false
34. since = null
35. argv = args()
36. i = 0
37. while (i < argv.length) {
38. if (argv[i] == '--limit' && i + 1 < argv.length) { limit = toNumber(argv[i + 1]) }
39. if (argv[i] == '--delta') { delta = true }
40. if (argv[i] == '--plan') { planOnly = true }
41. if (argv[i] == '--since' && i + 1 < argv.length) { since = argv[i + 1] }
42. i = i + 1
43. }
45. started = now()
46. totals = emptyTotals()
47. today = timestamp(started).slice(0, 10)
48. runMode = 'full'
49. changeRequests = 0
50. ids = []
51. if (delta) {
52. // --since: as if the last run had been on that day (the stored day stays as it is)
53. let p = planStart(today, since)
54. runMode = p.mode
55. console.log('tmdb sync: ' + p.mode + ' (' + p.why + ')')
56. if (p.mode == 'delta') {
57. let t0 = now()
58. let done = false
59. while (!done) {
60. let ps = planStep()
61. changeRequests = changeRequests + ps.requests + ps.tvmazeRequests
62. totals.requests = totals.requests + ps.requests
63. totals.tvmazeRequests = totals.tvmazeRequests + ps.tvmazeRequests
64. done = ps.done
65. if (ps.failed != '') { runMode = 'full' }
66. sleep(pauseMsAfter(ps) / 1000)
67. }
68. console.log('tmdb sync: ' + planLine() + ' in ' + ((now() - t0) / 1000) + ' s')
69. ids = runMode == 'delta' ? planIds() : allFollowedShowIds()
70. } else { ids = allFollowedShowIds() }
71. } else { ids = allFollowedShowIds() }
72. if (planOnly) {
73. console.log('tmdb sync plan: ' + ids.length + ' titles: ' + ids.join(' '))
74. exit(0)
75. }
76. console.log('tmdb sync: ' + ids.length + (runMode == 'delta' ? ' changed titles' : ' followed shows') + (limit >= 0 ? ', limit ' + limit : ''))
77. for (id of ids) {
78. if (limit < 0 || totals.shows < limit) {
79. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze — tmdbsync.hl syncShowDelta)
80. let light = runMode == 'delta' && !isFollowedInRun(id)
81. let r = light ? syncShowDelta(id) : syncShow(id)
82. totals = addTotals(totals, r)
83. // tracker#18 + #28: the record's own fields + the cast + crew from the same details answer
84. let c = r.details != null ? applySynced(id, r.details) : { renamed = false cast = 0 crew = 0 people = 0 error = '' }
85. console.log(' ' + id + (light ? ' (light)' : '') + (c.renamed ? ' (renamed)' : '') + ': +' + r.newSeasons + ' seasons +' + r.newEpisodes + ' episodes ~' + r.updatedEpisodes + ' updated, poster ' + r.posters + ', ids +' + r.ids
86. + ', tvmaze +' + r.tvmazeEpisodes + ' episodes +' + r.tvmazeSeasons + ' seasons ~' + r.tvmazeFilled + ' filled' + (r.tvmazeSkipped > 0 ? ' SKIPPED (numbered differently)' : '') + ', cast ' + c.cast + ' crew ' + c.crew + ' +' + c.people + ' people' + (r.error != '' ? ' ERROR ' + r.error : '') + (c.error != '' ? ' ERROR ' + c.error : ''))
87. if (totals.shows % 10 == 0) { persistDetails() }
88. sleep(pauseMsAfter(r) / 1000)
89. }
90. }
91. persistDetails()
92. if (delta && since == null && limit < 0 && runMode == 'delta') { markSynced(today) }
93. console.log('tmdb sync done: mode=' + runMode + ' changeRequests=' + changeRequests + ' ' + totalsLine(totals) + ' seconds=' + ((now() - started) / 1000))
95. on Error(e) { console.log(' error absorbed: ' + e.message) }