tracker

11.9 KB

1. // lib/dailysync.hl — WHICH TITLES THE DAILY SYNC ASKS FOR (tracker.worldapi.org#18; deltasync.hl before mission 028). Creator:
2. // "it checks every show in a request to the api? cause they have an endpoint that gives a delta on bulk". Before: 2 requests
3. // per FOLLOWED show every day (256 for 126 shows), the other titles never. Now the daily run (jobs.hl syncTick,
4. // tools/sync-tmdb.hl --delta) first asks what changed since the last run and syncs ONLY those titles of ours — followed or not:
5. //
6. // GET <TMDB_BASE_URL>/tv/changes?start_date=<last run day>&end_date=<today>&page=<n> changed TV ids, 100 per page
7. // GET <TMDB_BASE_URL>/movie/changes?start_date=…&end_date=…&page=<n> changed movie ids, 100 per page
8. // GET <TVMAZE_BASE_URL>/updates/shows?since=day|week|month { "<tvmaze id>": <updated at> }
9. //
10. // → intersected with our titles (search.hl's index: TMDB id by kind, TVmaze id — no table scan) → the sync's queue, FOLLOWED
11. // titles first. One change-list request per clock step (jobs.hl), 260 ms per TMDB / 550 ms per TVmaze request between them.
12. // A FOLLOWED title gets the full step (all seasons, TVmaze — as before #18); one NOBODY follows the light step (sync.hl
13. // syncShowDelta: details, ids, poster, record, cast + only the seasons that can have changed, no TVmaze) — so TVmaze's list
14. // only brings followed series — tracker#35 (mission 030): and, for a series NOBODY follows, its TVmaze STATUS (the page's pill
15. // prefers TVmaze's): named by TVmaze's list alone → a status-only step (sync.hl syncTvmazeStatus, one TVmaze request), named by
16. // both lists → the light step asks it too. `syncRunStep` picks the step.
17. // Left out: titles without a TMDB id, incomplete ones (shows.hl isIncomplete — the details repair completes them) and ADULT
18. // ones nobody follows (never public, their posters are not kept — mission 056).
19. //
20. // THE LAST RUN: `<storage>/sync-state.txt` = the day ('YYYY-MM-DD', UTC) the last finished run STARTED (its change window began
21. // at 00:00 of that day, so two runs overlap by a few hours, never leave a gap). FULL WALK (every followed show, as before #18)
22. // when there is no such day (the first run), when it is more than `maxDeltaDays` (14: TMDB's change lists reach 14 days back)
23. // ago, or when a change list cannot be read (then the day is NOT advanced: the next run asks for the missed days too).
24. import { fetch } from 'hl:fetch'
25. import { now } from 'hl:time'
26. import { readFile, writeFile, exists } from 'hl:fs'
27. import { storageDir, textOr, hasId } from './util.hl'
28. import { showsTable, isIncomplete } from './shows.hl'
29. import { tmdbGet, tvmazeBase, numIdOr } from './tmdb.hl'
30. import { syncShow, syncShowDelta, syncTvmazeStatus } from './sync.hl'
31. import { dayNumber } from './sync-helpers.hl'
32. import { allFollowedShowIds } from './follows.hl'
33. import { titleIdByTmdb, titleIdByTvmaze } from './search.hl'
35. static maxDeltaDays = 14
36. // TMDB answers at most 500 pages of a list
37. static maxPages = 500
38. static stateFile = storageDir + '/sync-state.txt'
40. // the day of the last finished run, or null
41. static lastSyncDay = () => {
42. if (!exists(stateFile)) { return null }
43. t = readFile(stateFile)
44. if (t == null || hlTypeName(t) != 'String') { return null }
45. d = t.trim()
46. return dayNumber(d) != null ? d.slice(0, 10) : null
47. }
48. static markSynced = (&day) => {
49. writeFile(stateFile, day, 420)
50. return true
51. }
53. // ---- the plan of one run (one run at a time: the app's clock or the tool) --------------------------------------------------
54. // flat statics written key by key (like people.hl's maps); `run.no` tells this run's entries from older ones
55. static run = { no = 0 mode = '' why = '' since = '' until = '' tvmazeSince = 'day' phase = 'done' page = 1 pages = 1
56. requests = 0 tvmazeRequests = 0 tvChanged = 0 movieChanged = 0 tvmazeChanged = 0 count = 0 followed = 0 statusOnly = 0 failed = '' }
57. static followedRun = {}
58. static pickedRun = {}
59. static pickedAt = {}
60. // tracker#35: named by TVmaze's change list (this run) · picked for its TVmaze status alone
61. static mazeRun = {}
62. static statusOnlyRun = {}
63. static followedList = { ids = [] }
65. // a run starts: { mode ('delta' | 'full'), why, since, days }. 'delta' → call planStep() until it answers done, then planIds().
66. // `sinceDay` (the tool's --since) stands in for the stored last day; null = the stored one
67. static planStart = (today, &sinceDay) => {
68. run.no = run.no + 1
69. run.since = ''
70. run.until = today
71. run.phase = 'done'
72. run.page = 1
73. run.pages = 1
74. run.requests = 0
75. run.tvmazeRequests = 0
76. run.tvChanged = 0
77. run.movieChanged = 0
78. run.tvmazeChanged = 0
79. run.count = 0
80. run.followed = 0
81. run.statusOnly = 0
82. run.failed = ''
83. last = sinceDay != null ? sinceDay : lastSyncDay()
84. days = last != null && dayNumber(today) != null ? dayNumber(today) - dayNumber(last) : null
85. if (last == null) {
86. run.mode = 'full'
87. run.why = 'no last run'
88. } else if (days == null || days > maxDeltaDays || days < 0) {
89. run.mode = 'full'
90. run.why = 'last run ' + last + ', ' + days + ' days ago > ' + maxDeltaDays
91. } else {
92. run.mode = 'delta'
93. run.why = 'since ' + last + ' (' + days + (days == 1 ? ' day' : ' days') + ')'
94. run.since = last
95. run.tvmazeSince = days <= 1 ? 'day' : (days <= 7 ? 'week' : 'month')
96. run.phase = 'tv'
97. ids = allFollowedShowIds()
98. for (id of ids) { followedRun[id] = run.no }
99. followedList.ids = ids
100. }
101. return { mode = run.mode why = run.why since = run.since days = days }
102. }
104. // is this title followed by anyone (as of this run's start)? — such a title gets the FULL step (all seasons + TVmaze, as
105. // before #18), one nobody follows the LIGHT one (sync.hl syncShowDelta: changed seasons, no TVmaze)
106. static isFollowedInRun = (&showId) => { return followedRun[showId] == run.no }
108. // one of ours that changed → into the queue once, when the daily sync may touch it. `viaTvmaze`: found by TVmaze's list — only
109. // a followed series counts there (TVmaze is merged for the followed ones only)
110. static pick = (showId, &viaTvmaze) => {
111. if (showId == null || pickedRun[showId] == run.no) { return false }
112. s = showsTable.fetch(showId)
113. followed = followedRun[showId] == run.no
114. if (s == null || s.tmdbId == null || isIncomplete(s) || (s.adult == true && !followed) || (viaTvmaze && !followed)) { return false }
115. pickedRun[showId] = run.no
116. pickedAt['' + run.count] = showId
117. run.count = run.count + 1
118. if (followed) { run.followed = run.followed + 1 }
119. return true
120. }
122. // tracker#35: a series nobody follows, only on TVmaze's list → into the queue for its TVmaze status alone (one picked by TMDB's
123. // list already gets it with its light step)
124. static pickStatus = (showId) => {
125. if (showId == null || pickedRun[showId] == run.no || followedRun[showId] == run.no) { return false }
126. s = showsTable.fetch(showId)
127. if (s == null || s.type != 'series' || !hasId(s.tvmzId) || s.adult == true || s.mergedInto != null || isIncomplete(s)) { return false }
128. pickedRun[showId] = run.no
129. statusOnlyRun[showId] = run.no
130. pickedAt['' + run.count] = showId
131. run.count = run.count + 1
132. run.statusOnly = run.statusOnly + 1
133. return true
134. }
136. // ONE title of the run (jobs.hl syncTick, tools/sync-tmdb.hl): a full walk or a followed title → the full step; a series only on
137. // TVmaze's list → its TVmaze status; any other → the light step (+ the TVmaze status when TVmaze's list named it too)
138. static syncRunStep = (&showId, &delta) => {
139. if (!delta || followedRun[showId] == run.no) { return syncShow(showId) }
140. if (statusOnlyRun[showId] == run.no) { return syncTvmazeStatus(showId) }
141. return syncShowDelta(showId, mazeRun[showId] == run.no)
142. }
143. static isStatusOnlyInRun = (&showId) => { return statusOnlyRun[showId] == run.no }
145. // ONE change-list request (TMDB tv page, TMDB movie page or TVmaze's list): { done, requests, tvmazeRequests, failed ('' = none) }
146. static planStep = () => {
147. out = { done = false requests = 0 tvmazeRequests = 0 failed = '' }
148. if (run.phase == 'tv' || run.phase == 'movie') {
149. kind = run.phase
150. out.requests = 1
151. run.requests = run.requests + 1
152. let j = tmdbGet('/' + kind + '/changes?start_date=' + run.since + '&end_date=' + run.until + '&page=' + run.page)
153. if (j.failed != null || j.results == null) {
154. run.failed = j.failed != null ? j.failed : 'no results in /' + kind + '/changes'
155. run.phase = 'done'
156. out.failed = run.failed
157. out.done = true
158. return out
159. }
160. type = kind == 'tv' ? 'series' : 'movie'
161. for (c of j.results) {
162. if (c != null && numIdOr(c.id) != null) {
163. if (kind == 'tv') { run.tvChanged = run.tvChanged + 1 } else { run.movieChanged = run.movieChanged + 1 }
164. let sid = titleIdByTmdb(type, c.id)
165. if (sid != null) { pick(sid, false) }
166. }
167. }
168. run.pages = j.total_pages != null && hlTypeName(j.total_pages) == 'Number' ? j.total_pages : 1
169. if (run.pages > maxPages) { run.pages = maxPages }
170. if (run.page < run.pages) { run.page = run.page + 1 } else {
171. run.phase = kind == 'tv' ? 'movie' : 'tvmaze'
172. run.page = 1
173. run.pages = 1
174. }
175. return out
176. }
177. if (run.phase == 'tvmaze') {
178. out.tvmazeRequests = 1
179. run.tvmazeRequests = run.tvmazeRequests + 1
180. run.phase = 'done'
181. f = fetch(tvmazeBase + '/updates/shows?since=' + run.tvmazeSince, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 30000 })
182. let j = f != null && f.status == 200 ? f.json() : null
183. if (j == null) {
184. run.failed = 'TVmaze: ' + (f == null || f.status == null || f.status == 0 ? 'no answer' : 'HTTP ' + f.status) + ' for /updates/shows?since=' + run.tvmazeSince
185. out.failed = run.failed
186. out.done = true
187. return out
188. }
189. for (k of j.keys()) {
190. run.tvmazeChanged = run.tvmazeChanged + 1
191. let sid = titleIdByTvmaze(k)
192. if (sid != null) {
193. mazeRun[sid] = run.no
194. if (!pick(sid, true)) { pickStatus(sid) }
195. }
196. }
197. }
198. out.done = true
199. return out
200. }
202. // the queue: the changed titles of ours, the followed ones first (in follow-table order), then the rest (in change-list order)
203. static planIds = () => {
204. out = []
205. for (id of followedList.ids) { if (pickedRun[id] == run.no) { out.push(id) } }
206. let i = 0
207. while (i < run.count) {
208. let id = pickedAt['' + i]
209. if (followedRun[id] != run.no) { out.push(id) }
210. i = i + 1
211. }
212. return out
213. }
215. // the log line of a finished plan
216. static planLine = () => {
217. return 'changes ' + run.why + ': tmdb tv=' + run.tvChanged + ' movie=' + run.movieChanged + ' tvmaze(' + run.tvmazeSince + ')=' + run.tvmazeChanged
218. + ' → ' + run.count + ' of our titles (' + run.followed + ' followed' + (run.statusOnly > 0 ? ', ' + run.statusOnly + ' TVmaze status only' : '') + '), changeRequests=' + run.requests + ' tvmazeChangeRequests=' + run.tvmazeRequests
219. + (run.failed != '' ? ' FAILED: ' + run.failed : '')
220. }
222. // ---- the copied summaries (tracker#18, one time; mission 070: MOVED, never deleted) -------------------------------------------
223. // The migration copied TMDB's text into `summary` too (9,648 of 10,011 titles on the live copy, 2026-10-01): such a `summary` is
224. // no text of the creator's — it is MOVED to `migratedSummary` (kept in the record, shown nowhere) and `summary` is emptied, so the
225. // CURRENT tmdbSummary shows (shows.hl summaryOf). Nothing is lost: the conductor's rule (mission 070) — no step destroys migrated
226. // data. A `summary` that differs is the creator's own and stays. ONCE: `<storage>/summaries-moved.txt` records that it ran (and
227. // the counts). → { ran, moved, kept, errors, ms }
228. static movedFile = storageDir + '/summaries-moved.txt'
229. static moveCopiedSummaries = () => {
230. out = { ran = false moved = 0 kept = 0 errors = 0 ms = 0 }
231. if (exists(movedFile)) { return out }
232. out.ran = true
233. t0 = now()
234. for (s of showsTable.find(null, null)) {
235. if (textOr(s.summary) != null) {
236. if (s.summary == s.tmdbSummary) {
237. let x = s
238. x.migratedSummary = s.summary
239. x.summary = null
240. if (showsTable.update(x.id, x) == null) { out.errors = out.errors + 1 } else { out.moved = out.moved + 1 }
241. } else { out.kept = out.kept + 1 }
242. }
243. }
244. showsTable.persist()
245. out.ms = now() - t0
246. writeFile(movedFile, 'moved=' + out.moved + ' kept=' + out.kept + ' errors=' + out.errors, 420)
247. return out
248. }