tracker

12.1 KB

1. // lib/franchiseseed.hl — THE CURATED FRANCHISES (tracker.worldapi.org#40, mission 034). Creator: "you never build timlines and
2. // franchises". The big franchises (Star Trek, Star Wars, Stargate, MCU, DC …) with their timelines, MOVIES AND SERIES, come from
3. // a hand-written DATA file in the repo — `data/franchises.json` (TRACKER_FRANCHISE_FILE): per franchise its name, summary,
4. // the TMDB collections that join it, and its timelines, each with its order ("in-universe" | "release") and its titles as
5. // [kind ('tv' | 'movie'), TMDB id, "title (year)" — for people only].
6. //
7. // franchiseSeedTable pk @id { kind, key, ref, order, franchise, created, why, at } — what the seed made, one row per thing:
8. // kind 'franchise' key = its name ref = the franchise @id created (true: the seed made it; false: the
9. // creator's own of that name, adopted)
10. // kind 'timeline' key = franchise|timeline ref = the timeline @id order + franchise = what the seed LAST WROTE
11. // (order null: the creator's own of that name)
12. // kind 'collection' key = collection:<id> ref = the timeline @id franchise (null: it had one already, left alone)
13. // kind 'missing' key = tv:<id> / movie:<id> why TMDB has no such title (404) — not asked again
14. // storage/mpackdb/franchiseseed.db
15. //
16. // THE JOB (jobs.hl `franchiseSeedTick`, after every other start job — the collection seed last — one step per tick):
17. // 1. the titles of the file we don't have (search.hl titleIdByTmdb) are IMPORTED, one per step, through the search's own
18. // import (details.hl importWithCredits: details, seasons + episodes, poster, ids, TVmaze, cast + crew — complete), then
19. // the pause the daily sync makes after the same requests (tmdb.hl pauseMsAfter). Without TMDB_READ_TOKEN: no imports.
20. // 2. then ONE build step (no TMDB): each franchise found or made, its collections' timelines attached, each timeline made
21. // or brought up to date with the titles we have, in the file's order.
22. // EDITOR CHANGES WIN: a seeded timeline whose order or franchise is no longer what the seed last wrote was edited by the
23. // creator — it is never written again (`kept`); a franchise or timeline of the same name the creator made is used / left
24. // alone, never doubled; a collection timeline that has a franchise keeps it. RESUMES after a restart (imports are stored at
25. // once; the queue = what is still missing); IDEMPOTENT (a second start imports nothing and writes nothing).
26. import { MPackDB } from 'hl:mpackdb'
27. import { readFile, exists } from 'hl:fs'
28. import { now } from 'hl:time'
29. import { storageDir, envOr, newId } from './util.hl'
30. import { showById, liveShowOf } from './shows.hl'
31. import { titleIdByTmdb } from './search.hl'
32. import { importWithCredits } from './details.hl'
33. import { ensureFranchises, franchiseById, timelineById, createFranchise, putTimeline, setTimelineFranchise, franchiseByName, timelineByName, timelineOfCollection, storeSeeded, persistFranchises } from './franchises.hl'
35. static franchiseSeedTable = new MPackDB(file = storageDir + '/franchiseseed.db', primaryKey = '@id', indexes = [])
36. static franchiseFile = envOr('TRACKER_FRANCHISE_FILE', './data/franchises.json')
38. // the queue is a plain static list filled in place (a static cannot be reassigned — collections.hl)
39. static importQueue = []
40. static seedRows = {}
41. static fseed = { built = false pos = 0 file = false franchises = 0 timelines = 0 titles = 0 have = 0 imported = 0 failed = 0 missing = 0
42. requests = 0 tvmazeRequests = 0 franchisesCreated = 0 franchisesAdopted = 0 timelinesCreated = 0 updated = 0 unchanged = 0 kept = 0 empty = 0
43. placed = 0 collections = 0 writes = 0 error = '' }
45. static typeOfKind = (&kind) => { return kind == 'tv' ? 'series' : 'movie' }
46. static titleKeyOf = (&kind, &tmdbId) => { return kind + ':' + tmdbId }
47. // our show @id for an entry of the file (a merged tombstone answers its keeper), or null
48. static ourIdOf = (&entry) => {
49. id = titleIdByTmdb(typeOfKind(entry[0]), entry[1])
50. s = id != null ? liveShowOf(showById(id)) : null
51. return s != null ? s.id : null
52. }
53. static sameList = (&a, &b) => {
54. if (a == null || b == null || a.length != b.length) { return false }
55. let i = 0
56. while (i < a.length) {
57. if (a[i] != b[i]) { return false }
58. i = i + 1
59. }
60. return true
61. }
62. static noteRow = (kind, key, &ref, &order, &franchise, created, why) => {
63. old = seedRows[kind + '|' + key]
64. rec = { id = old != null ? old.id : newId() kind = kind key = key ref = ref order = order franchise = franchise created = created why = why at = now() }
65. if (old != null) { franchiseSeedTable.update(rec.id, rec) } else { franchiseSeedTable.put(rec) }
66. seedRows[kind + '|' + key] = rec
67. return rec
68. }
69. static rowOf = (&kind, &key) => { return seedRows[kind + '|' + key] }
71. // the file → { franchises: [...] }, or null (logged) when it is not there
72. static readFranchiseFile = () => {
73. if (!exists(franchiseFile)) {
74. fseed.error = franchiseFile + ' not found'
75. return null
76. }
77. d = JSON.parse(readFile(franchiseFile))
78. if (d == null || d.franchises == null) {
79. fseed.error = franchiseFile + ': no "franchises" list'
80. return null
81. }
82. return d
83. }
84. static fileData = { franchises = [] }
86. // at the job's start: the file read, the seed's rows loaded, the queue = the file's titles we don't have (`imports` false: none)
87. static franchiseSeedPrepare = (&imports) => {
88. if (fseed.built) { return importQueue.length }
89. fseed.built = true
90. ensureFranchises()
91. for (r of franchiseSeedTable.find(null, null)) { seedRows[r.kind + '|' + r.key] = r }
92. d = readFranchiseFile()
93. if (d == null) { return 0 }
94. fseed.file = true
95. fileData.franchises = d.franchises
96. queued = {}
97. for (f of d.franchises) {
98. fseed.franchises = fseed.franchises + 1
99. for (t of f.timelines) {
100. fseed.timelines = fseed.timelines + 1
101. for (e of t.titles) {
102. let key = titleKeyOf(e[0], e[1])
103. if (queued[key] != true) {
104. queued[key] = true
105. fseed.titles = fseed.titles + 1
106. if (ourIdOf(e) != null) { fseed.have = fseed.have + 1 }
107. else if (rowOf('missing', key) != null) { fseed.missing = fseed.missing + 1 }
108. else if (imports) { importQueue.push(e) }
109. }
110. }
111. }
112. }
113. return importQueue.length
114. }
115. static franchiseSeedStats = () => { return fseed }
117. // ---- step 1: ONE import → { requests, tvmazeRequests (what the pause counts), error } --------------------------------------
118. static importOne = (&entry) => {
119. out = { requests = 1 tvmazeRequests = 0 error = '' done = false }
120. key = titleKeyOf(entry[0], entry[1])
121. fseed.requests = fseed.requests + 1
122. r = importWithCredits(entry[0], entry[1])
123. if (r.error != null) {
124. fseed.failed = fseed.failed + 1
125. out.error = key + ' (' + entry[2] + '): ' + r.error
126. // TMDB has no such title: remembered, never asked again (a passing failure is tried again on the next start)
127. if (r.error.includes('HTTP 404')) { noteRow('missing', key, null, null, null, false, r.error) franchiseSeedTable.persist() }
128. return out
129. }
130. if (r.sync != null) {
131. out.requests = 1 + r.sync.requests
132. out.tvmazeRequests = r.sync.tvmazeRequests
133. fseed.requests = fseed.requests + r.sync.requests
134. fseed.tvmazeRequests = fseed.tvmazeRequests + r.sync.tvmazeRequests
135. }
136. if (r.created == true) { fseed.imported = fseed.imported + 1 } else { fseed.have = fseed.have + 1 }
137. return out
138. }
140. // ---- step 2: the build (no TMDB) ---------------------------------------------------------------------------------------------
141. // the franchise of the file's entry `f`: the one the seed made / adopted before, else the creator's of that name, else a new one
142. static franchiseOf = (&f) => {
143. row = rowOf('franchise', f.name)
144. if (row != null) { return franchiseById(row.ref) }
145. mine = franchiseByName(f.name)
146. if (mine != null) {
147. fseed.franchisesAdopted = fseed.franchisesAdopted + 1
148. noteRow('franchise', f.name, mine.id, null, null, false, 'the creator\'s own')
149. return mine
150. }
151. c = createFranchise(f.name, f.summary)
152. if (c.error != null) {
153. fseed.error = f.name + ': ' + c.error
154. return null
155. }
156. fseed.franchisesCreated = fseed.franchisesCreated + 1
157. noteRow('franchise', f.name, c.id, null, null, true, '')
158. return franchiseById(c.id)
159. }
160. // a TMDB collection's timeline (collection seed) joins the franchise — once, and only when it has none yet
161. static attachCollection = (&collection, &fid) => {
162. key = 'collection:' + collection
163. if (rowOf('collection', key) != null) { return false }
164. t = timelineOfCollection(collection)
165. if (t == null) { return false }
166. if (t.franchise != null && t.franchise != '') {
167. noteRow('collection', key, t.id, null, null, false, 'has a franchise already')
168. return false
169. }
170. r = setTimelineFranchise(t.id, fid)
171. if (r.error != null) {
172. fseed.error = key + ': ' + r.error
173. return false
174. }
175. fseed.collections = fseed.collections + 1
176. noteRow('collection', key, t.id, null, fid, true, '')
177. return true
178. }
179. // one timeline of the file: made, brought up to date, or left alone (the creator's)
180. static buildTimeline = (&f, &fid, &tl) => {
181. key = f.name + '|' + tl.name
182. order = []
183. for (e of tl.titles) {
184. let sid = ourIdOf(e)
185. if (sid != null && !order.includes(sid)) { order.push(sid) }
186. }
187. fseed.placed = fseed.placed + order.length
188. row = rowOf('timeline', key)
189. if (row == null) {
190. mine = timelineByName(tl.name)
191. if (mine != null) {
192. // the creator made one of that name: theirs, never written by the seed
193. fseed.kept = fseed.kept + 1
194. noteRow('timeline', key, mine.id, null, null, false, 'the creator\'s own')
195. return true
196. }
197. // none of its titles here yet (no TMDB token, or every import failed): made on a later start
198. if (order.length == 0) {
199. fseed.empty = fseed.empty + 1
200. return true
201. }
202. rec = putTimeline(tl.name, fid, order, 'seed', null, tl.order == 'release' ? 'release' : 'in-universe')
203. if (rec == null) {
204. fseed.error = key + ': not stored'
205. return false
206. }
207. fseed.timelinesCreated = fseed.timelinesCreated + 1
208. fseed.writes = fseed.writes + 1
209. noteRow('timeline', key, rec.id, order, fid, true, '')
210. return true
211. }
212. t = timelineById(row.ref)
213. // gone, or the creator's own, or edited since the seed wrote it (its order or franchise changed): editor changes win
214. if (t == null || row.order == null || !sameList(t.order, row.order) || t.franchise != row.franchise) {
215. fseed.kept = fseed.kept + 1
216. return true
217. }
218. if (sameList(t.order, order) && t.franchise == fid) {
219. fseed.unchanged = fseed.unchanged + 1
220. return true
221. }
222. r = storeSeeded(t, order, fid)
223. if (r.error != null) {
224. fseed.error = key + ': ' + r.error
225. return false
226. }
227. fseed.updated = fseed.updated + 1
228. fseed.writes = fseed.writes + 1
229. noteRow('timeline', key, t.id, order, fid, true, '')
230. return true
231. }
232. static buildAll = () => {
233. for (f of fileData.franchises) {
234. let fr = franchiseOf(f)
235. if (fr != null) {
236. if (f.collections != null) { for (c of f.collections) { attachCollection(c, fr.id) } }
237. for (tl of f.timelines) { buildTimeline(f, fr.id, tl) }
238. }
239. }
240. persistFranchises()
241. franchiseSeedTable.persist()
242. return true
243. }
245. // ONE step: the next import, or — the queue done — the build → { requests, tvmazeRequests, error, done }
246. static franchiseSeedStep = () => {
247. if (!fseed.file) { return { requests = 0 tvmazeRequests = 0 error = fseed.error done = true } }
248. if (fseed.pos < importQueue.length) {
249. e = importQueue[fseed.pos]
250. fseed.pos = fseed.pos + 1
251. // imported meanwhile (the search, a person page): nothing to ask
252. if (ourIdOf(e) != null) {
253. fseed.have = fseed.have + 1
254. return { requests = 0 tvmazeRequests = 0 error = '' done = false }
255. }
256. return importOne(e)
257. }
258. buildAll()
259. return { requests = 0 tvmazeRequests = 0 error = fseed.error done = true }
260. }
261. static franchiseSeedLine = () => {
262. s = fseed
263. return 'franchises=' + s.franchises + ' (created=' + s.franchisesCreated + ' adopted=' + s.franchisesAdopted + ') timelines=' + s.timelines + ' (created=' + s.timelinesCreated
264. + ' updated=' + s.updated + ' unchanged=' + s.unchanged + ' kept=' + s.kept + ' empty=' + s.empty + ') collections=' + s.collections + ' titles=' + s.titles + ' had=' + s.have
265. + ' imported=' + s.imported + ' failed=' + s.failed + ' missing=' + s.missing + ' placed=' + s.placed + ' requests=' + s.requests + ' tvmazeRequests=' + s.tvmazeRequests
266. }