tracker

9.8 KB

1. // deltasync.hl — WHICH TITLES THE DAILY SYNC ASKS FOR (tracker.worldapi.org#18). Creator: "it checks every show in a request to
2. // the api? cause they have an endpoint that gives a delta on bulk". Before: 2 requests per FOLLOWED show every day (256 for 126
3. // shows), the other titles never. Now the daily run (project.hl syncTick, tools/sync-tmdb.hl --delta) first asks what changed
4. // since the last run and syncs ONLY those titles of ours — followed or not:
5. //
6. // GET <TMDB_BASE_URL>/tv/changes?start_date=<last run day>&end_date=<today>&page=<n> changed TV ids, 100 per page
7. // GET <TMDB_BASE_URL>/movie/changes?start_date=…&end_date=…&page=<n> changed movie ids, 100 per page
8. // GET <TVMAZE_BASE_URL>/updates/shows?since=day|week|month { "<tvmaze id>": <updated at> }
9. //
10. // → intersected with our titles (search.hl's index: TMDB id by kind, TVmaze id — no table scan) → the sync's queue, FOLLOWED
11. // titles first. One change-list request per clock step (project.hl), 260 ms per TMDB / 550 ms per TVmaze request between them.
12. // A FOLLOWED title gets the full step (all seasons, TVmaze — as before #18); one NOBODY follows the light step (tmdbsync.hl
13. // syncShowDelta: details, ids, poster, record, cast + only the seasons that can have changed, no TVmaze) — so TVmaze's list
14. // only brings followed series.
15. // Left out: titles without a TMDB id, incomplete ones (details.hl isIncomplete — the details repair completes them) and ADULT
16. // ones nobody follows (never public, their posters are not kept — mission 056).
17. //
18. // THE LAST RUN: `<storage>/sync-state.txt` = the day ('YYYY-MM-DD', UTC) the last finished run STARTED (its change window began
19. // at 00:00 of that day, so two runs overlap by a few hours, never leave a gap). FULL WALK (every followed show, as before #18)
20. // when there is no such day (the first run), when it is more than `maxDeltaDays` (14: TMDB's change lists reach 14 days back)
21. // ago, or when a change list cannot be read (then the day is NOT advanced: the next run asks for the missed days too).
22. import { fetch } from 'hl:fetch'
23. import { now } from 'hl:time'
24. import { readFile, writeFile, exists } from 'hl:fs'
25. import { showsTable, storageDir } from './shows.hl'
26. import { tmdbGet, tvmazeBase, textOr, numIdOr, dayNumber } from './tmdbsync.hl'
27. import { isIncomplete } from './details.hl'
28. import { allFollowedShowIds } from './follows.hl'
29. import { titleIdByTmdb, titleIdByTvmaze } from './search.hl'
31. static maxDeltaDays = 14
32. // TMDB answers at most 500 pages of a list
33. static maxPages = 500
34. static stateFile = storageDir + '/sync-state.txt'
36. // the day of the last finished run, or null
37. static lastSyncDay = () => {
38. if (!exists(stateFile)) { return null }
39. let t = readFile(stateFile)
40. if (t == null || hlTypeName(t) != 'String') { return null }
41. let d = t.trim()
42. return dayNumber(d) != null ? d.slice(0, 10) : null
43. }
44. static markSynced = (day) => {
45. writeFile(stateFile, day, 420)
46. return true
47. }
49. // ---- the plan of one run (one run at a time: the app's clock or the tool) --------------------------------------------------
50. // flat statics written key by key (like people.hl's maps); `run.no` tells this run's entries from older ones
51. static run = { no = 0 mode = '' why = '' since = '' until = '' tvmazeSince = 'day' phase = 'done' page = 1 pages = 1
52. requests = 0 tvmazeRequests = 0 tvChanged = 0 movieChanged = 0 tvmazeChanged = 0 count = 0 followed = 0 failed = '' }
53. static followedRun = {}
54. static pickedRun = {}
55. static pickedAt = {}
56. static followedList = { ids = [] }
58. // a run starts: { mode ('delta' | 'full'), why, since, days }. 'delta' → call planStep() until it answers done, then planIds().
59. // `sinceDay` (the tool's --since) stands in for the stored last day; null = the stored one
60. static planStart = (today, sinceDay) => {
61. run.no = run.no + 1
62. run.since = ''
63. run.until = today
64. run.phase = 'done'
65. run.page = 1
66. run.pages = 1
67. run.requests = 0
68. run.tvmazeRequests = 0
69. run.tvChanged = 0
70. run.movieChanged = 0
71. run.tvmazeChanged = 0
72. run.count = 0
73. run.followed = 0
74. run.failed = ''
75. let last = sinceDay != null ? sinceDay : lastSyncDay()
76. let days = last != null && dayNumber(today) != null ? dayNumber(today) - dayNumber(last) : null
77. if (last == null) {
78. run.mode = 'full'
79. run.why = 'no last run'
80. } else if (days == null || days > maxDeltaDays || days < 0) {
81. run.mode = 'full'
82. run.why = 'last run ' + last + ', ' + days + ' days ago > ' + maxDeltaDays
83. } else {
84. run.mode = 'delta'
85. run.why = 'since ' + last + ' (' + days + (days == 1 ? ' day' : ' days') + ')'
86. run.since = last
87. run.tvmazeSince = days <= 1 ? 'day' : (days <= 7 ? 'week' : 'month')
88. run.phase = 'tv'
89. let ids = allFollowedShowIds()
90. for (id of ids) { followedRun[id] = run.no }
91. followedList.ids = ids
92. }
93. return { mode = run.mode why = run.why since = run.since days = days }
94. }
96. // is this title followed by anyone (as of this run's start)? — such a title gets the FULL step (all seasons + TVmaze, as
97. // before #18), one nobody follows the LIGHT one (tmdbsync.hl syncShowDelta: changed seasons, no TVmaze)
98. static isFollowedInRun = (showId) => { return followedRun[showId] == run.no }
100. // one of ours that changed → into the queue once, when the daily sync may touch it. `viaTvmaze`: found by TVmaze's list — only
101. // a followed series counts there (TVmaze is merged for the followed ones only)
102. static pick = (showId, viaTvmaze) => {
103. if (showId == null || pickedRun[showId] == run.no) { return false }
104. let s = showsTable.fetch(showId)
105. let followed = followedRun[showId] == run.no
106. if (s == null || s.tmdbId == null || isIncomplete(s) || (s.adult == true && !followed) || (viaTvmaze && !followed)) { return false }
107. pickedRun[showId] = run.no
108. pickedAt['' + run.count] = showId
109. run.count = run.count + 1
110. if (followed) { run.followed = run.followed + 1 }
111. return true
112. }
114. // ONE change-list request (TMDB tv page, TMDB movie page or TVmaze's list): { done, requests, tvmazeRequests, failed ('' = none) }
115. static planStep = () => {
116. let out = { done = false requests = 0 tvmazeRequests = 0 failed = '' }
117. if (run.phase == 'tv' || run.phase == 'movie') {
118. let kind = run.phase
119. out.requests = 1
120. run.requests = run.requests + 1
121. let j = tmdbGet('/' + kind + '/changes?start_date=' + run.since + '&end_date=' + run.until + '&page=' + run.page)
122. if (j.failed != null || j.results == null) {
123. run.failed = j.failed != null ? j.failed : 'no results in /' + kind + '/changes'
124. run.phase = 'done'
125. out.failed = run.failed
126. out.done = true
127. return out
128. }
129. let type = kind == 'tv' ? 'series' : 'movie'
130. for (c of j.results) {
131. if (c != null && numIdOr(c.id) != null) {
132. if (kind == 'tv') { run.tvChanged = run.tvChanged + 1 } else { run.movieChanged = run.movieChanged + 1 }
133. let sid = titleIdByTmdb(type, c.id)
134. if (sid != null) { pick(sid, false) }
135. }
136. }
137. run.pages = j.total_pages != null && hlTypeName(j.total_pages) == 'Number' ? j.total_pages : 1
138. if (run.pages > maxPages) { run.pages = maxPages }
139. if (run.page < run.pages) { run.page = run.page + 1 } else {
140. run.phase = kind == 'tv' ? 'movie' : 'tvmaze'
141. run.page = 1
142. run.pages = 1
143. }
144. return out
145. }
146. if (run.phase == 'tvmaze') {
147. out.tvmazeRequests = 1
148. run.tvmazeRequests = run.tvmazeRequests + 1
149. run.phase = 'done'
150. let f = fetch(tvmazeBase + '/updates/shows?since=' + run.tvmazeSince, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 30000 })
151. let j = f != null && f.status == 200 ? f.json() : null
152. if (j == null) {
153. run.failed = 'TVmaze: ' + (f == null || f.status == null || f.status == 0 ? 'no answer' : 'HTTP ' + f.status) + ' for /updates/shows?since=' + run.tvmazeSince
154. out.failed = run.failed
155. out.done = true
156. return out
157. }
158. for (k of j.keys()) {
159. run.tvmazeChanged = run.tvmazeChanged + 1
160. let sid = titleIdByTvmaze(k)
161. if (sid != null) { pick(sid, true) }
162. }
163. }
164. out.done = true
165. return out
166. }
168. // the queue: the changed titles of ours, the followed ones first (in follow-table order), then the rest (in change-list order)
169. static planIds = () => {
170. let out = []
171. for (id of followedList.ids) { if (pickedRun[id] == run.no) { out.push(id) } }
172. let i = 0
173. while (i < run.count) {
174. let id = pickedAt['' + i]
175. if (followedRun[id] != run.no) { out.push(id) }
176. i = i + 1
177. }
178. return out
179. }
181. // the log line of a finished plan
182. static planLine = () => {
183. return 'changes ' + run.why + ': tmdb tv=' + run.tvChanged + ' movie=' + run.movieChanged + ' tvmaze(' + run.tvmazeSince + ')=' + run.tvmazeChanged
184. + ' → ' + run.count + ' of our titles (' + run.followed + ' followed), changeRequests=' + run.requests + ' tvmazeChangeRequests=' + run.tvmazeRequests
185. + (run.failed != '' ? ' FAILED: ' + run.failed : '')
186. }
188. // ---- the copied summaries (tracker#18, one time) ---------------------------------------------------------------------------
189. // The migration copied TMDB's text into `summary` too (9,648 of 10,011 titles on the live copy, 2026-10-01): such a `summary` is
190. // no text of the creator's — it is cleared, so the CURRENT tmdbSummary shows (shows.hl summaryOf). A `summary` that differs is
191. // the creator's own and stays. ONCE: `<storage>/summaries-cleared.txt` records that it ran (and the counts). → { ran, cleared, kept }
192. static clearedFile = storageDir + '/summaries-cleared.txt'
193. static clearCopiedSummaries = () => {
194. let out = { ran = false cleared = 0 kept = 0 errors = 0 ms = 0 }
195. if (exists(clearedFile)) { return out }
196. out.ran = true
197. let t0 = now()
198. for (s of showsTable.find(null, null)) {
199. if (textOr(s.summary) != null) {
200. if (s.summary == s.tmdbSummary) {
201. let x = s
202. x.summary = null
203. if (showsTable.update(x.id, x) == null) { out.errors = out.errors + 1 } else { out.cleared = out.cleared + 1 }
204. } else { out.kept = out.kept + 1 }
205. }
206. }
207. showsTable.persist()
208. out.ms = now() - t0
209. writeFile(clearedFile, 'cleared=' + out.cleared + ' kept=' + out.kept + ' errors=' + out.errors, 420)
210. return out
211. }