tracker

10.0 KB

1. // lib/dailysync.hl — WHICH TITLES THE DAILY SYNC ASKS FOR (tracker.worldapi.org#18; deltasync.hl before mission 028). Creator:
2. // "it checks every show in a request to the api? cause they have an endpoint that gives a delta on bulk". Before: 2 requests
3. // per FOLLOWED show every day (256 for 126 shows), the other titles never. Now the daily run (jobs.hl syncTick,
4. // tools/sync-tmdb.hl --delta) first asks what changed since the last run and syncs ONLY those titles of ours — followed or not:
5. //
6. // GET <TMDB_BASE_URL>/tv/changes?start_date=<last run day>&end_date=<today>&page=<n> changed TV ids, 100 per page
7. // GET <TMDB_BASE_URL>/movie/changes?start_date=…&end_date=…&page=<n> changed movie ids, 100 per page
8. // GET <TVMAZE_BASE_URL>/updates/shows?since=day|week|month { "<tvmaze id>": <updated at> }
9. //
10. // → intersected with our titles (search.hl's index: TMDB id by kind, TVmaze id — no table scan) → the sync's queue, FOLLOWED
11. // titles first. One change-list request per clock step (jobs.hl), 260 ms per TMDB / 550 ms per TVmaze request between them.
12. // A FOLLOWED title gets the full step (all seasons, TVmaze — as before #18); one NOBODY follows the light step (sync.hl
13. // syncShowDelta: details, ids, poster, record, cast + only the seasons that can have changed, no TVmaze) — so TVmaze's list
14. // only brings followed series.
15. // Left out: titles without a TMDB id, incomplete ones (shows.hl isIncomplete — the details repair completes them) and ADULT
16. // ones nobody follows (never public, their posters are not kept — mission 056).
17. //
18. // THE LAST RUN: `<storage>/sync-state.txt` = the day ('YYYY-MM-DD', UTC) the last finished run STARTED (its change window began
19. // at 00:00 of that day, so two runs overlap by a few hours, never leave a gap). FULL WALK (every followed show, as before #18)
20. // when there is no such day (the first run), when it is more than `maxDeltaDays` (14: TMDB's change lists reach 14 days back)
21. // ago, or when a change list cannot be read (then the day is NOT advanced: the next run asks for the missed days too).
22. import { fetch } from 'hl:fetch'
23. import { now } from 'hl:time'
24. import { readFile, writeFile, exists } from 'hl:fs'
25. import { storageDir, textOr } from './util.hl'
26. import { showsTable, isIncomplete } from './shows.hl'
27. import { tmdbGet, tvmazeBase, numIdOr } from './tmdb.hl'
28. import { dayNumber } from './sync-helpers.hl'
29. import { allFollowedShowIds } from './follows.hl'
30. import { titleIdByTmdb, titleIdByTvmaze } from './search.hl'
32. static maxDeltaDays = 14
33. // TMDB answers at most 500 pages of a list
34. static maxPages = 500
35. static stateFile = storageDir + '/sync-state.txt'
37. // the day of the last finished run, or null
38. static lastSyncDay = () => {
39. if (!exists(stateFile)) { return null }
40. t = readFile(stateFile)
41. if (t == null || hlTypeName(t) != 'String') { return null }
42. d = t.trim()
43. return dayNumber(d) != null ? d.slice(0, 10) : null
44. }
45. static markSynced = (&day) => {
46. writeFile(stateFile, day, 420)
47. return true
48. }
50. // ---- the plan of one run (one run at a time: the app's clock or the tool) --------------------------------------------------
51. // flat statics written key by key (like people.hl's maps); `run.no` tells this run's entries from older ones
52. static run = { no = 0 mode = '' why = '' since = '' until = '' tvmazeSince = 'day' phase = 'done' page = 1 pages = 1
53. requests = 0 tvmazeRequests = 0 tvChanged = 0 movieChanged = 0 tvmazeChanged = 0 count = 0 followed = 0 failed = '' }
54. static followedRun = {}
55. static pickedRun = {}
56. static pickedAt = {}
57. static followedList = { ids = [] }
59. // a run starts: { mode ('delta' | 'full'), why, since, days }. 'delta' → call planStep() until it answers done, then planIds().
60. // `sinceDay` (the tool's --since) stands in for the stored last day; null = the stored one
61. static planStart = (today, &sinceDay) => {
62. run.no = run.no + 1
63. run.since = ''
64. run.until = today
65. run.phase = 'done'
66. run.page = 1
67. run.pages = 1
68. run.requests = 0
69. run.tvmazeRequests = 0
70. run.tvChanged = 0
71. run.movieChanged = 0
72. run.tvmazeChanged = 0
73. run.count = 0
74. run.followed = 0
75. run.failed = ''
76. last = sinceDay != null ? sinceDay : lastSyncDay()
77. days = last != null && dayNumber(today) != null ? dayNumber(today) - dayNumber(last) : null
78. if (last == null) {
79. run.mode = 'full'
80. run.why = 'no last run'
81. } else if (days == null || days > maxDeltaDays || days < 0) {
82. run.mode = 'full'
83. run.why = 'last run ' + last + ', ' + days + ' days ago > ' + maxDeltaDays
84. } else {
85. run.mode = 'delta'
86. run.why = 'since ' + last + ' (' + days + (days == 1 ? ' day' : ' days') + ')'
87. run.since = last
88. run.tvmazeSince = days <= 1 ? 'day' : (days <= 7 ? 'week' : 'month')
89. run.phase = 'tv'
90. ids = allFollowedShowIds()
91. for (id of ids) { followedRun[id] = run.no }
92. followedList.ids = ids
93. }
94. return { mode = run.mode why = run.why since = run.since days = days }
95. }
97. // is this title followed by anyone (as of this run's start)? — such a title gets the FULL step (all seasons + TVmaze, as
98. // before #18), one nobody follows the LIGHT one (sync.hl syncShowDelta: changed seasons, no TVmaze)
99. static isFollowedInRun = (&showId) => { return followedRun[showId] == run.no }
101. // one of ours that changed → into the queue once, when the daily sync may touch it. `viaTvmaze`: found by TVmaze's list — only
102. // a followed series counts there (TVmaze is merged for the followed ones only)
103. static pick = (showId, &viaTvmaze) => {
104. if (showId == null || pickedRun[showId] == run.no) { return false }
105. s = showsTable.fetch(showId)
106. followed = followedRun[showId] == run.no
107. if (s == null || s.tmdbId == null || isIncomplete(s) || (s.adult == true && !followed) || (viaTvmaze && !followed)) { return false }
108. pickedRun[showId] = run.no
109. pickedAt['' + run.count] = showId
110. run.count = run.count + 1
111. if (followed) { run.followed = run.followed + 1 }
112. return true
113. }
115. // ONE change-list request (TMDB tv page, TMDB movie page or TVmaze's list): { done, requests, tvmazeRequests, failed ('' = none) }
116. static planStep = () => {
117. out = { done = false requests = 0 tvmazeRequests = 0 failed = '' }
118. if (run.phase == 'tv' || run.phase == 'movie') {
119. kind = run.phase
120. out.requests = 1
121. run.requests = run.requests + 1
122. let j = tmdbGet('/' + kind + '/changes?start_date=' + run.since + '&end_date=' + run.until + '&page=' + run.page)
123. if (j.failed != null || j.results == null) {
124. run.failed = j.failed != null ? j.failed : 'no results in /' + kind + '/changes'
125. run.phase = 'done'
126. out.failed = run.failed
127. out.done = true
128. return out
129. }
130. type = kind == 'tv' ? 'series' : 'movie'
131. for (c of j.results) {
132. if (c != null && numIdOr(c.id) != null) {
133. if (kind == 'tv') { run.tvChanged = run.tvChanged + 1 } else { run.movieChanged = run.movieChanged + 1 }
134. let sid = titleIdByTmdb(type, c.id)
135. if (sid != null) { pick(sid, false) }
136. }
137. }
138. run.pages = j.total_pages != null && hlTypeName(j.total_pages) == 'Number' ? j.total_pages : 1
139. if (run.pages > maxPages) { run.pages = maxPages }
140. if (run.page < run.pages) { run.page = run.page + 1 } else {
141. run.phase = kind == 'tv' ? 'movie' : 'tvmaze'
142. run.page = 1
143. run.pages = 1
144. }
145. return out
146. }
147. if (run.phase == 'tvmaze') {
148. out.tvmazeRequests = 1
149. run.tvmazeRequests = run.tvmazeRequests + 1
150. run.phase = 'done'
151. f = fetch(tvmazeBase + '/updates/shows?since=' + run.tvmazeSince, { headers = { 'accept' = 'application/json' 'user-agent' = 'tracker.worldapi.org (tmdb sync)' } timeoutMs = 30000 })
152. let j = f != null && f.status == 200 ? f.json() : null
153. if (j == null) {
154. run.failed = 'TVmaze: ' + (f == null || f.status == null || f.status == 0 ? 'no answer' : 'HTTP ' + f.status) + ' for /updates/shows?since=' + run.tvmazeSince
155. out.failed = run.failed
156. out.done = true
157. return out
158. }
159. for (k of j.keys()) {
160. run.tvmazeChanged = run.tvmazeChanged + 1
161. let sid = titleIdByTvmaze(k)
162. if (sid != null) { pick(sid, true) }
163. }
164. }
165. out.done = true
166. return out
167. }
169. // the queue: the changed titles of ours, the followed ones first (in follow-table order), then the rest (in change-list order)
170. static planIds = () => {
171. out = []
172. for (id of followedList.ids) { if (pickedRun[id] == run.no) { out.push(id) } }
173. let i = 0
174. while (i < run.count) {
175. let id = pickedAt['' + i]
176. if (followedRun[id] != run.no) { out.push(id) }
177. i = i + 1
178. }
179. return out
180. }
182. // the log line of a finished plan
183. static planLine = () => {
184. return 'changes ' + run.why + ': tmdb tv=' + run.tvChanged + ' movie=' + run.movieChanged + ' tvmaze(' + run.tvmazeSince + ')=' + run.tvmazeChanged
185. + ' → ' + run.count + ' of our titles (' + run.followed + ' followed), changeRequests=' + run.requests + ' tvmazeChangeRequests=' + run.tvmazeRequests
186. + (run.failed != '' ? ' FAILED: ' + run.failed : '')
187. }
189. // ---- the copied summaries (tracker#18, one time; mission 070: MOVED, never deleted) -------------------------------------------
190. // The migration copied TMDB's text into `summary` too (9,648 of 10,011 titles on the live copy, 2026-10-01): such a `summary` is
191. // no text of the creator's — it is MOVED to `migratedSummary` (kept in the record, shown nowhere) and `summary` is emptied, so the
192. // CURRENT tmdbSummary shows (shows.hl summaryOf). Nothing is lost: the conductor's rule (mission 070) — no step destroys migrated
193. // data. A `summary` that differs is the creator's own and stays. ONCE: `<storage>/summaries-moved.txt` records that it ran (and
194. // the counts). → { ran, moved, kept, errors, ms }
195. static movedFile = storageDir + '/summaries-moved.txt'
196. static moveCopiedSummaries = () => {
197. out = { ran = false moved = 0 kept = 0 errors = 0 ms = 0 }
198. if (exists(movedFile)) { return out }
199. out.ran = true
200. t0 = now()
201. for (s of showsTable.find(null, null)) {
202. if (textOr(s.summary) != null) {
203. if (s.summary == s.tmdbSummary) {
204. let x = s
205. x.migratedSummary = s.summary
206. x.summary = null
207. if (showsTable.update(x.id, x) == null) { out.errors = out.errors + 1 } else { out.moved = out.moved + 1 }
208. } else { out.kept = out.kept + 1 }
209. }
210. }
211. showsTable.persist()
212. out.ms = now() - t0
213. writeFile(movedFile, 'moved=' + out.moved + ' kept=' + out.kept + ' errors=' + out.errors, 420)
214. return out
215. }