tracker

5.4 KB

1. // tools/sync-tmdb.hl — THE TMDB SYNC FROM THE COMMAND LINE (tracker.worldapi.org#9): the same per-show sync
2. // the app runs every day (lib/sync.hl), for every followed show, one after the other, waiting between shows
3. // so TMDB's 40 requests / 10 s (and TVmaze's 20 / 10 s) are never exceeded. Idempotent: a second run adds nothing (prints 0 new).
4. //
5. // STOP THE APP FIRST (or run it on a COPY): hl:mpackdb does not coordinate two processes on one storage
6. // directory (hybriel#21). The running app does this itself every day (jobs.hl `syncTick`) — this tool is
7. // for tests, the real-data proof and a manual catch-up.
8. //
9. // set -a; . ./.env; set +a # TMDB_READ_TOKEN — never print it
10. // TRACKER_STORAGE=/abs/path/storage/mpackdb ./bin/hybriel tools/sync-tmdb.hl [--limit N] [--delta [--since YYYY-MM-DD] [--plan]]
11. //
12. // Default: the FULL WALK (every followed show). --delta (tracker#18): what the app's daily run does — the change lists since
13. // the last run (dailysync.hl; `--since` overrides the stored day), then only the changed titles of ours; stores the day at the
14. // end like the app (not with --since or --limit). --plan: only the change lists and the queue they give, no title synced.
15. //
16. // Prints one line per show and the totals: shows, newSeasons, newEpisodes, updatedEpisodes, updatedSeasons,
17. // posters, ids (external ids filled, tracker.worldapi.org#12), requests, tvmazeRequests, errors, seconds, and (#15) what
18. // TVmaze added: tvmazeEpisodes, tvmazeSeasons, tvmazeFilled (episodes whose empty title/air date it filled), tvmazeSkipped
19. // (shows TVmaze numbers differently, left alone).
20. import { env, args, exit } from 'hl:proc'
21. import { now, sleep, timestamp } from 'hl:time'
22. import { syncEnabled, pauseMsAfter } from '../lib/tmdb.hl'
23. import { persistSync } from '../lib/sync.hl'
24. import { emptyTotals, addTotals, totalsLine } from '../lib/sync-helpers.hl'
25. import { allFollowedShowIds } from '../lib/follows.hl'
26. import { applySynced, persistDetails } from '../lib/details.hl'
27. import { planStart, planStep, planIds, planLine, markSynced, isFollowedInRun, isStatusOnlyInRun, syncRunStep } from '../lib/dailysync.hl'
29. storageDir = env('TRACKER_STORAGE')
30. if (storageDir == null || !storageDir.startsWith('/')) { console.log('REFUSED: set TRACKER_STORAGE, absolute (app stopped, or a copy)') exit(2) }
31. if (!syncEnabled) { console.log('REFUSED: TMDB_READ_TOKEN is not set') exit(2) }
33. limit = -1
34. delta = false
35. planOnly = false
36. since = null
37. argv = args()
38. i = 0
39. while (i < argv.length) {
40. if (argv[i] == '--limit' && i + 1 < argv.length) { limit = toNumber(argv[i + 1]) }
41. if (argv[i] == '--delta') { delta = true }
42. if (argv[i] == '--plan') { planOnly = true }
43. if (argv[i] == '--since' && i + 1 < argv.length) { since = argv[i + 1] }
44. i = i + 1
45. }
47. started = now()
48. totals = emptyTotals()
49. today = timestamp(started).slice(0, 10)
50. runMode = 'full'
51. changeRequests = 0
52. ids = []
53. if (delta) {
54. // --since: as if the last run had been on that day (the stored day stays as it is)
55. p = planStart(today, since)
56. runMode = p.mode
57. console.log('tmdb sync: ' + p.mode + ' (' + p.why + ')')
58. if (p.mode == 'delta') {
59. t0 = now()
60. let done = false
61. while (!done) {
62. let ps = planStep()
63. changeRequests = changeRequests + ps.requests + ps.tvmazeRequests
64. totals.requests = totals.requests + ps.requests
65. totals.tvmazeRequests = totals.tvmazeRequests + ps.tvmazeRequests
66. done = ps.done
67. if (ps.failed != '') { runMode = 'full' }
68. sleep(pauseMsAfter(ps) / 1000)
69. }
70. console.log('tmdb sync: ' + planLine() + ' in ' + ((now() - t0) / 1000) + ' s')
71. ids = runMode == 'delta' ? planIds() : allFollowedShowIds()
72. } else { ids = allFollowedShowIds() }
73. } else { ids = allFollowedShowIds() }
74. if (planOnly) {
75. console.log('tmdb sync plan: ' + ids.length + ' titles: ' + ids.join(' '))
76. exit(0)
77. }
78. console.log('tmdb sync: ' + ids.length + (runMode == 'delta' ? ' changed titles' : ' followed shows') + (limit >= 0 ? ', limit ' + limit : ''))
79. for (id of ids) {
80. if (limit < 0 || totals.shows < limit) {
81. // tracker#18: in a delta run a title nobody follows gets the light step (changed seasons, no TVmaze — sync.hl syncShowDelta);
82. // tracker#35: one only on TVmaze's change list its TVmaze status (dailysync.hl syncRunStep)
83. let light = runMode == 'delta' && !isFollowedInRun(id)
84. let r = syncRunStep(id, runMode == 'delta')
85. totals = addTotals(totals, r)
86. // tracker#18 + #28: the record's own fields + the cast + crew from the same details answer
87. let c = r.details != null ? applySynced(id, r.details) : { renamed = false cast = 0 crew = 0 people = 0 error = '' }
88. console.log(' ' + id + (isStatusOnlyInRun(id) ? ' (TVmaze status)' : (light ? ' (light)' : '')) + (c.renamed ? ' (renamed)' : '') + ': +' + r.newSeasons + ' seasons +' + r.newEpisodes + ' episodes ~' + r.updatedEpisodes + ' updated, poster ' + r.posters + ', ids +' + r.ids
89. + ', tvmaze +' + r.tvmazeEpisodes + ' episodes +' + r.tvmazeSeasons + ' seasons ~' + r.tvmazeFilled + ' filled' + (r.tvmazeSkipped > 0 ? ' SKIPPED (numbered differently)' : '') + ', cast ' + c.cast + ' crew ' + c.crew + ' +' + c.people + ' people' + (r.error != '' ? ' ERROR ' + r.error : '') + (c.error != '' ? ' ERROR ' + c.error : ''))
90. if (totals.shows % 10 == 0) { persistDetails() }
91. sleep(pauseMsAfter(r) / 1000)
92. }
93. }
94. persistDetails()
95. if (delta && since == null && limit < 0 && runMode == 'delta') { markSynced(today) }
96. console.log('tmdb sync done: mode=' + runMode + ' changeRequests=' + changeRequests + ' ' + totalsLine(totals) + ' seconds=' + ((now() - started) / 1000))
98. on Error(e) { console.log(' error absorbed: ' + e.message) }