tracker

6.9 KB

1. // lib/collections.hl — TIMELINES FROM TMDB'S MOVIE COLLECTIONS (tracker.worldapi.org#19, mission 066): the seed of franchises.hl's
2. // timelines. Mission 028: out of franchises.hl.
3. //
4. // collectionChecksTable pk = the movie's show @id { collection (TMDB id | null), why, at } — the seed's progress
5. // storage/mpackdb/collectionchecks.db
6. //
7. // THE SEED (TMDB movie collections): every movie with a tmdbId (adult ones skipped) is asked once
8. // GET <TMDB>/movie/<tmdbId> → belongs_to_collection { id, name } (or null)
9. // GET <TMDB>/collection/<id> → parts [{ id, title, release_date, adult }] — once per collection
10. // A collection with at least TWO of our (non-adult) movies becomes one timeline (no franchise), its parts in release order;
11. // every part we have is marked checked at once (no own details request). A collection with fewer is remembered (checks) and
12. // gives no timeline (a widget with both ends disabled says nothing). Run in the app like the adult backfill (jobs.hl
13. // `collectionTick`, one movie per step, ≤ 40 TMDB requests / 10 s, waits while the daily sync or the adult backfill runs),
14. // RESUMES after a restart (the queue = the movies without a check row). A hand-edited timeline is never touched by it.
15. import { MPackDB } from 'hl:mpackdb'
16. import { now } from 'hl:time'
17. import { storageDir, dateOr, textOr } from './util.hl'
18. import { showsTable } from './shows.hl'
19. import { tmdbGet } from './tmdb.hl'
20. import { timelinesTable, ensureFranchises, putTimeline, seededCoverOf, setSuperseded } from './franchises.hl'
22. static collectionChecksTable = new MPackDB(file = storageDir + '/collectionchecks.db', primaryKey = '@id', indexes = [])
24. // the queue is a plain static list filled in place (a static cannot be reassigned; read through a field it was copied on every
25. // access — shortids.hl), worked off from `seed.pos`
26. static seedQueue = []
27. static checkedShows = {}
28. static collectionsDone = {}
29. static movieByTmdb = {}
30. static seedTries = {}
31. static seed = { built = false pos = 0 movies = 0 checked = 0 requests = 0 timelines = 0 titles = 0 superseded = 0 small = 0 noCollection = 0 notFound = 0 errors = 0 retries = 0 }
33. static markChecked = (&showId, collection, why) => {
34. if (checkedShows[showId] == true) { return true }
35. checkedShows[showId] = true
36. seed.checked = seed.checked + 1
37. if (collectionChecksTable.put({ id = showId collection = collection why = why at = now() }) == null) {
38. console.log('collections: check of ' + showId + ' not stored: ' + collectionChecksTable.lastError())
39. }
40. return true
41. }
42. static seedPrepare = () => {
43. if (seed.built) { return seedQueue.length }
44. seed.built = true
45. ensureFranchises()
46. for (c of collectionChecksTable.find(null, null)) {
47. checkedShows[c.id] = true
48. if (c.collection != null) { collectionsDone['' + c.collection] = true }
49. }
50. for (t of timelinesTable.find(null, null)) { if (t.tmdbCollection != null) { collectionsDone['' + t.tmdbCollection] = true } }
51. for (s of showsTable.find(null, null)) {
52. if (s.type == 'movie' && s.tmdbId != null && s.adult != true) {
53. movieByTmdb['' + s.tmdbId] = s.id
54. if (checkedShows[s.id] != true) { seedQueue.push(s.id) }
55. }
56. }
57. seed.movies = seedQueue.length
58. return seedQueue.length
59. }
60. static seedLeft = () => { return seedQueue.length - seed.pos }
61. static seedStats = () => { return seed }
62. static seedPersist = () => {
63. collectionChecksTable.persist()
64. timelinesTable.persist()
65. return true
66. }
67. // a passing failure goes to the back of the queue, at most 2 more tries per start
68. static seedRetry = (&showId) => {
69. n = seedTries[showId] != null ? seedTries[showId] : 0
70. if (n >= 2) { return false }
71. seedTries[showId] = n + 1
72. seedQueue.push(showId)
73. seed.retries = seed.retries + 1
74. return true
75. }
77. // ONE movie: → { requests (TMDB API requests made: what the pause counts), done, error }
78. static seedStep = () => {
79. out = { requests = 0 done = false error = '' }
80. seedPrepare()
81. if (seed.pos >= seedQueue.length) { out.done = true return out }
82. id = seedQueue[seed.pos]
83. seed.pos = seed.pos + 1
84. if (checkedShows[id] == true) { return out }
85. show = showsTable.fetch(id)
86. if (show == null || show.adult == true || show.tmdbId == null) { return out }
87. out.requests = 1
88. seed.requests = seed.requests + 1
89. d = tmdbGet('/movie/' + show.tmdbId)
90. if (d.failed != null) {
91. seed.errors = seed.errors + 1
92. out.error = show.title + ': ' + d.failed
93. if (d.status == 404) { seed.notFound = seed.notFound + 1 markChecked(id, null, 'TMDB 404') } else { seedRetry(id) }
94. return out
95. }
96. c = d.belongs_to_collection
97. if (c == null || c.id == null) {
98. seed.noCollection = seed.noCollection + 1
99. markChecked(id, null, 'no collection')
100. return out
101. }
102. cid = '' + c.id
103. if (collectionsDone[cid] == true) { markChecked(id, c.id, 'collection known') return out }
104. out.requests = 2
105. seed.requests = seed.requests + 1
106. coll = tmdbGet('/collection/' + cid)
107. if (coll.failed != null || coll.parts == null) {
108. seed.errors = seed.errors + 1
109. out.error = show.title + ': collection ' + cid + ': ' + (coll.failed != null ? coll.failed : 'no parts')
110. if (coll.status == 404) { markChecked(id, c.id, 'collection 404') collectionsDone[cid] = true } else { seedRetry(id) }
111. return out
112. }
113. // the parts in release order (no date last), only the non-adult ones we have
114. let parts = []
115. for (p of coll.parts) {
116. let next = []
117. let done = false
118. let pd = dateOr(p.release_date)
119. for (x of parts) {
120. if (!done && pd != '' && (x.d == '' || pd < x.d)) { next.push({ p = p d = pd }) done = true }
121. next.push(x)
122. }
123. if (!done) { next.push({ p = p d = pd }) }
124. parts = next
125. }
126. order = []
127. for (x of parts) {
128. let sid = x.p.adult == true ? null : movieByTmdb['' + x.p.id]
129. if (sid != null && !order.includes(sid)) { order.push(sid) }
130. }
131. collectionsDone[cid] = true
132. if (order.length >= 2) {
133. name = textOr(coll.name) != null ? coll.name.trim() : (textOr(c.name) != null ? c.name.trim() : 'Collection ' + cid)
134. // tracker#40: all of it in one curated timeline → made hidden (superseded, no second widget); partly → joins that franchise
135. cov = seededCoverOf(order)
136. rec = putTimeline(name, cov.franchise, order, 'tmdb', c.id, 'release')
137. if (rec == null) {
138. seed.errors = seed.errors + 1
139. out.error = 'timeline "' + name + '" not stored: ' + timelinesTable.lastError()
140. } else {
141. seed.timelines = seed.timelines + 1
142. seed.titles = seed.titles + order.length
143. if (cov.cover != null) {
144. setSuperseded(rec, cov.cover)
145. seed.superseded = seed.superseded + 1
146. }
147. }
148. } else { seed.small = seed.small + 1 }
149. for (sid of order) { markChecked(sid, c.id, 'collection') }
150. markChecked(id, c.id, 'collection')
151. return out
152. }
153. static seedLine = () => {
154. return seed.pos + '/' + seedQueue.length + ' movies, checked=' + seed.checked + ' timelines=' + seed.timelines + ' titles=' + seed.titles + ' superseded=' + seed.superseded + ' small=' + seed.small
155. + ' noCollection=' + seed.noCollection + ' notFound=' + seed.notFound + ' requests=' + seed.requests + ' errors=' + seed.errors + ' retries=' + seed.retries
156. }