tracker

13.5 KB

1. // lib/franchiseseed.hl — THE CURATED FRANCHISES (tracker.worldapi.org#40, mission 034). Creator: "you never build timlines and
2. // franchises". The big franchises (Star Trek, Star Wars, Stargate, MCU, DC …) with their timelines, MOVIES AND SERIES, come from
3. // a hand-written DATA file in the repo — `data/franchises.json` (TRACKER_FRANCHISE_FILE): per franchise its name, summary,
4. // the TMDB collections that join it, and its timelines, each with its order ("in-universe" | "release") and its titles as
5. // [kind ('tv' | 'movie'), TMDB id, "title (year)" — for people only].
6. //
7. // franchiseSeedTable pk @id { kind, key, ref, order, franchise, created, why, at } — what the seed made, one row per thing:
8. // kind 'franchise' key = its name ref = the franchise @id created (true: the seed made it; false: the
9. // creator's own of that name, adopted)
10. // kind 'timeline' key = franchise|timeline ref = the timeline @id order + franchise = what the seed LAST WROTE
11. // (order null: the creator's own of that name)
12. // kind 'collection' key = collection:<id> ref = the timeline @id franchise (null: it had one already, left alone)
13. // kind 'missing' key = tv:<id> / movie:<id> why TMDB has no such title (404) — not asked again
14. // storage/mpackdb/franchiseseed.db
15. //
16. // THE JOB (jobs.hl `franchiseSeedTick`, after every other start job — the collection seed last — one step per tick):
17. // 1. the titles of the file we don't have (search.hl titleIdByTmdb) are IMPORTED, one per step, through the search's own
18. // import (details.hl importWithCredits: details, seasons + episodes, poster, ids, TVmaze, cast + crew — complete), then
19. // the pause the daily sync makes after the same requests (tmdb.hl pauseMsAfter). Without TMDB_READ_TOKEN: no imports.
20. // 2. then ONE build step (no TMDB): each franchise found or made, its collections' timelines attached, each timeline made
21. // or brought up to date with the titles we have, in the file's order; then every TMDB collection timeline whose titles
22. // are ALL in one curated timeline is hidden (`supersededBy`, franchises.hl seededCoverOf), a partly covered one joins
23. // that franchise.
24. // EDITOR CHANGES WIN: a seeded timeline whose order or franchise is no longer what the seed last wrote was edited by the
25. // creator — it is never written again (`kept`); a franchise or timeline of the same name the creator made is used / left
26. // alone, never doubled; a collection timeline that has a franchise keeps it. RESUMES after a restart (imports are stored at
27. // once; the queue = what is still missing); IDEMPOTENT (a second start imports nothing and writes nothing).
28. import { MPackDB } from 'hl:mpackdb'
29. import { readFile, exists } from 'hl:fs'
30. import { now } from 'hl:time'
31. import { storageDir, envOr, newId } from './util.hl'
32. import { showById, liveShowOf } from './shows.hl'
33. import { titleIdByTmdb } from './search.hl'
34. import { importWithCredits } from './details.hl'
35. import { ensureFranchises, franchiseById, timelineById, timelinesTable, createFranchise, putTimeline, setTimelineFranchise, franchiseByName, timelineByName, timelineOfCollection, storeSeeded, persistFranchises, seededCoverOf, setSuperseded } from './franchises.hl'
37. static franchiseSeedTable = new MPackDB(file = storageDir + '/franchiseseed.db', primaryKey = '@id', indexes = [])
38. static franchiseFile = envOr('TRACKER_FRANCHISE_FILE', './data/franchises.json')
40. // the queue is a plain static list filled in place (a static cannot be reassigned — collections.hl)
41. static importQueue = []
42. static seedRows = {}
43. static fseed = { built = false pos = 0 file = false franchises = 0 timelines = 0 titles = 0 have = 0 imported = 0 failed = 0 missing = 0
44. requests = 0 tvmazeRequests = 0 franchisesCreated = 0 franchisesAdopted = 0 timelinesCreated = 0 updated = 0 unchanged = 0 kept = 0 empty = 0
45. placed = 0 collections = 0 superseded = 0 unsuperseded = 0 joined = 0 writes = 0 error = '' }
47. static typeOfKind = (&kind) => { return kind == 'tv' ? 'series' : 'movie' }
48. static titleKeyOf = (&kind, &tmdbId) => { return kind + ':' + tmdbId }
49. // our show @id for an entry of the file (a merged tombstone answers its keeper), or null
50. static ourIdOf = (&entry) => {
51. id = titleIdByTmdb(typeOfKind(entry[0]), entry[1])
52. s = id != null ? liveShowOf(showById(id)) : null
53. return s != null ? s.id : null
54. }
55. static sameList = (&a, &b) => {
56. if (a == null || b == null || a.length != b.length) { return false }
57. let i = 0
58. while (i < a.length) {
59. if (a[i] != b[i]) { return false }
60. i = i + 1
61. }
62. return true
63. }
64. static noteRow = (kind, key, &ref, &order, &franchise, created, why) => {
65. old = seedRows[kind + '|' + key]
66. rec = { id = old != null ? old.id : newId() kind = kind key = key ref = ref order = order franchise = franchise created = created why = why at = now() }
67. if (old != null) { franchiseSeedTable.update(rec.id, rec) } else { franchiseSeedTable.put(rec) }
68. seedRows[kind + '|' + key] = rec
69. return rec
70. }
71. static rowOf = (&kind, &key) => { return seedRows[kind + '|' + key] }
73. // the file → { franchises: [...] }, or null (logged) when it is not there
74. static readFranchiseFile = () => {
75. if (!exists(franchiseFile)) {
76. fseed.error = franchiseFile + ' not found'
77. return null
78. }
79. d = JSON.parse(readFile(franchiseFile))
80. if (d == null || d.franchises == null) {
81. fseed.error = franchiseFile + ': no "franchises" list'
82. return null
83. }
84. return d
85. }
86. static fileData = { franchises = [] }
88. // at the job's start: the file read, the seed's rows loaded, the queue = the file's titles we don't have (`imports` false: none)
89. static franchiseSeedPrepare = (&imports) => {
90. if (fseed.built) { return importQueue.length }
91. fseed.built = true
92. ensureFranchises()
93. for (r of franchiseSeedTable.find(null, null)) { seedRows[r.kind + '|' + r.key] = r }
94. d = readFranchiseFile()
95. if (d == null) { return 0 }
96. fseed.file = true
97. fileData.franchises = d.franchises
98. queued = {}
99. for (f of d.franchises) {
100. fseed.franchises = fseed.franchises + 1
101. for (t of f.timelines) {
102. fseed.timelines = fseed.timelines + 1
103. for (e of t.titles) {
104. let key = titleKeyOf(e[0], e[1])
105. if (queued[key] != true) {
106. queued[key] = true
107. fseed.titles = fseed.titles + 1
108. if (ourIdOf(e) != null) { fseed.have = fseed.have + 1 }
109. else if (rowOf('missing', key) != null) { fseed.missing = fseed.missing + 1 }
110. else if (imports) { importQueue.push(e) }
111. }
112. }
113. }
114. }
115. return importQueue.length
116. }
117. static franchiseSeedStats = () => { return fseed }
119. // ---- step 1: ONE import → { requests, tvmazeRequests (what the pause counts), error } --------------------------------------
120. static importOne = (&entry) => {
121. out = { requests = 1 tvmazeRequests = 0 error = '' done = false }
122. key = titleKeyOf(entry[0], entry[1])
123. fseed.requests = fseed.requests + 1
124. r = importWithCredits(entry[0], entry[1])
125. if (r.error != null) {
126. fseed.failed = fseed.failed + 1
127. out.error = key + ' (' + entry[2] + '): ' + r.error
128. // TMDB has no such title: remembered, never asked again (a passing failure is tried again on the next start)
129. if (r.error.includes('HTTP 404')) { noteRow('missing', key, null, null, null, false, r.error) franchiseSeedTable.persist() }
130. return out
131. }
132. if (r.sync != null) {
133. out.requests = 1 + r.sync.requests
134. out.tvmazeRequests = r.sync.tvmazeRequests
135. fseed.requests = fseed.requests + r.sync.requests
136. fseed.tvmazeRequests = fseed.tvmazeRequests + r.sync.tvmazeRequests
137. }
138. if (r.created == true) { fseed.imported = fseed.imported + 1 } else { fseed.have = fseed.have + 1 }
139. return out
140. }
142. // ---- step 2: the build (no TMDB) ---------------------------------------------------------------------------------------------
143. // the franchise of the file's entry `f`: the one the seed made / adopted before, else the creator's of that name, else a new one
144. static franchiseOf = (&f) => {
145. row = rowOf('franchise', f.name)
146. if (row != null) { return franchiseById(row.ref) }
147. mine = franchiseByName(f.name)
148. if (mine != null) {
149. fseed.franchisesAdopted = fseed.franchisesAdopted + 1
150. noteRow('franchise', f.name, mine.id, null, null, false, 'the creator\'s own')
151. return mine
152. }
153. c = createFranchise(f.name, f.summary)
154. if (c.error != null) {
155. fseed.error = f.name + ': ' + c.error
156. return null
157. }
158. fseed.franchisesCreated = fseed.franchisesCreated + 1
159. noteRow('franchise', f.name, c.id, null, null, true, '')
160. return franchiseById(c.id)
161. }
162. // a TMDB collection's timeline (collection seed) joins the franchise — once, and only when it has none yet
163. static attachCollection = (&collection, &fid) => {
164. key = 'collection:' + collection
165. if (rowOf('collection', key) != null) { return false }
166. t = timelineOfCollection(collection)
167. if (t == null) { return false }
168. if (t.franchise != null && t.franchise != '') {
169. noteRow('collection', key, t.id, null, null, false, 'has a franchise already')
170. return false
171. }
172. r = setTimelineFranchise(t.id, fid)
173. if (r.error != null) {
174. fseed.error = key + ': ' + r.error
175. return false
176. }
177. fseed.collections = fseed.collections + 1
178. noteRow('collection', key, t.id, null, fid, true, '')
179. return true
180. }
181. // one timeline of the file: made, brought up to date, or left alone (the creator's)
182. static buildTimeline = (&f, &fid, &tl) => {
183. key = f.name + '|' + tl.name
184. order = []
185. for (e of tl.titles) {
186. let sid = ourIdOf(e)
187. if (sid != null && !order.includes(sid)) { order.push(sid) }
188. }
189. fseed.placed = fseed.placed + order.length
190. row = rowOf('timeline', key)
191. if (row == null) {
192. mine = timelineByName(tl.name)
193. if (mine != null) {
194. // the creator made one of that name: theirs, never written by the seed
195. fseed.kept = fseed.kept + 1
196. noteRow('timeline', key, mine.id, null, null, false, 'the creator\'s own')
197. return true
198. }
199. // none of its titles here yet (no TMDB token, or every import failed): made on a later start
200. if (order.length == 0) {
201. fseed.empty = fseed.empty + 1
202. return true
203. }
204. rec = putTimeline(tl.name, fid, order, 'seed', null, tl.order == 'release' ? 'release' : 'in-universe')
205. if (rec == null) {
206. fseed.error = key + ': not stored'
207. return false
208. }
209. fseed.timelinesCreated = fseed.timelinesCreated + 1
210. fseed.writes = fseed.writes + 1
211. noteRow('timeline', key, rec.id, order, fid, true, '')
212. return true
213. }
214. t = timelineById(row.ref)
215. // gone, or the creator's own, or edited since the seed wrote it (its order or franchise changed): editor changes win
216. if (t == null || row.order == null || !sameList(t.order, row.order) || t.franchise != row.franchise) {
217. fseed.kept = fseed.kept + 1
218. return true
219. }
220. if (sameList(t.order, order) && t.franchise == fid) {
221. fseed.unchanged = fseed.unchanged + 1
222. return true
223. }
224. r = storeSeeded(t, order, fid)
225. if (r.error != null) {
226. fseed.error = key + ': ' + r.error
227. return false
228. }
229. fseed.updated = fseed.updated + 1
230. fseed.writes = fseed.writes + 1
231. noteRow('timeline', key, t.id, order, fid, true, '')
232. return true
233. }
234. // tracker#40: every TMDB collection timeline against the curated ones — all its titles in one → hidden (superseded: no second
235. // widget on First Contact); the cover gone → shown again; partly covered and without a franchise → joins that franchise
236. static supersedeCollections = () => {
237. for (t of timelinesTable.find(null, null)) {
238. if (t.tmdbCollection != null && t.order != null) {
239. let cov = seededCoverOf(t.order)
240. if (cov.cover != null && t.supersededBy != cov.cover) {
241. if (setSuperseded(t, cov.cover).error == null) { fseed.superseded = fseed.superseded + 1 }
242. } else if (cov.cover == null && t.supersededBy != null) {
243. if (setSuperseded(t, null).error == null) { fseed.unsuperseded = fseed.unsuperseded + 1 }
244. } else if (cov.cover == null && cov.franchise != null && (t.franchise == null || t.franchise == '')) {
245. if (setTimelineFranchise(t.id, cov.franchise).error == null) { fseed.joined = fseed.joined + 1 }
246. }
247. }
248. }
249. return true
250. }
251. static buildAll = () => {
252. for (f of fileData.franchises) {
253. let fr = franchiseOf(f)
254. if (fr != null) {
255. if (f.collections != null) { for (c of f.collections) { attachCollection(c, fr.id) } }
256. for (tl of f.timelines) { buildTimeline(f, fr.id, tl) }
257. }
258. }
259. supersedeCollections()
260. persistFranchises()
261. franchiseSeedTable.persist()
262. return true
263. }
265. // ONE step: the next import, or — the queue done — the build → { requests, tvmazeRequests, error, done }
266. static franchiseSeedStep = () => {
267. if (!fseed.file) { return { requests = 0 tvmazeRequests = 0 error = fseed.error done = true } }
268. if (fseed.pos < importQueue.length) {
269. e = importQueue[fseed.pos]
270. fseed.pos = fseed.pos + 1
271. // imported meanwhile (the search, a person page): nothing to ask
272. if (ourIdOf(e) != null) {
273. fseed.have = fseed.have + 1
274. return { requests = 0 tvmazeRequests = 0 error = '' done = false }
275. }
276. return importOne(e)
277. }
278. buildAll()
279. return { requests = 0 tvmazeRequests = 0 error = fseed.error done = true }
280. }
281. static franchiseSeedLine = () => {
282. s = fseed
283. return 'franchises=' + s.franchises + ' (created=' + s.franchisesCreated + ' adopted=' + s.franchisesAdopted + ') timelines=' + s.timelines + ' (created=' + s.timelinesCreated
284. + ' updated=' + s.updated + ' unchanged=' + s.unchanged + ' kept=' + s.kept + ' empty=' + s.empty + ') collections=' + s.collections + ' superseded=' + s.superseded + ' unsuperseded=' + s.unsuperseded + ' joined=' + s.joined + ' titles=' + s.titles + ' had=' + s.have
285. + ' imported=' + s.imported + ' failed=' + s.failed + ' missing=' + s.missing + ' placed=' + s.placed + ' requests=' + s.requests + ' tvmazeRequests=' + s.tvmazeRequests
286. }