tracker

8.4 KB

1. // lib/shortids.hl — SHORT IDS (tracker.worldapi.org#27, mission 060): every movie, series and person has its own 5-character id
2. // (`shortId`, like `a9s9a`), shown below its poster/photo; `/<shortId>` answers 301 to its page. Creator: "i wanted to give all
3. // our movies, shows, series and persons own short ids like a9s9a … those ids show below the poster."
4. //
5. // OLD IDS: the old tracker had them on 702 of 9,453 shows (Show.jsonl field `id`, base62 counter '100jh'…'1012y', all of
6. // them [a-z0-9]{5}). `data/old-short-ids.json` (made once by tools/old-short-ids.hl) maps the old Mongo `_id` (= our
7. // `oldId`, tools/migrate.hl) → that id; such a show gets it back. Persons never had one.
8. // NEW IDS: 5 random characters [a-z0-9] (36^5 = 60 M) — never one that is taken or one of the old ids (all 702 are
9. // reserved, even where the show is gone).
10. // UNIQUE ACROSS SHOWS AND PERSONS: one in-memory map `owners` (short id → { kind, id }) over BOTH tables — an mpackdb index
11. // is per table and cannot span two. Built at start by `ensureShortIds` (one scan of each table); every id handed
12. // out goes through it. A record whose short id another record already owns (never seen; logged) gets a new one.
13. // NEW RECORDS get one when they are created: `claimShortId(kind, id, oldId)` in search.hl importTitle and people.hl
14. // addMinimalTitle (the only places the app creates shows) and people.hl castPersonId (mission 062: the only place
15. // the app creates persons — the cast/crew of #26/#28: completion, credits job, daily sync + tools/sync-tmdb.hl,
16. // search import; and a guest star opened for the first time, people.hl guestPersonId).
17. // THE BACKFILL: the records without one (at first every row: 9,453 shows + 15,157 persons) are given theirs in the
18. // background, a time-boxed batch per clock tick (jobs.hl `shortIdTick` → `shortIdStep`), so pages are served
19. // between two batches; after a restart it goes on with what is still missing. Real data: STATUS.md mission 060.
20. //
21. // ONE PROCESS PER STORAGE (hybriel#21): the map lives in the app's process; the tables are shows.hl's.
22. import { now } from 'hl:time'
23. import { env } from 'hl:proc'
24. import { readFile, exists } from 'hl:fs'
25. import { randomBytes } from 'hl:crypto'
26. import { hexDigits } from './util.hl'
27. import { showsTable, personsTable, titlePath, liveShowOf } from './shows.hl'
29. static shortIdChars = 'abcdefghijklmnopqrstuvwxyz0123456789'
31. // short id → { kind ('show' | 'person'), id }
32. static owners = {}
33. // the old tracker's ids: oldId → short id, and every one of them → true (reserved)
34. static oldShortIds = {}
35. static reserved = {}
36. // the records still without a short id, [{ kind, id }], worked off from `state.queuePos`. Filled in place (push) and never
37. // reassigned (a static cannot be) — kept OUT of `state`: reading `state.queue[i]` copied the whole 25k list on every access
38. // (measured: ~3 ms per record instead of ~0.2)
39. static queue = []
40. static state = { built = false queuePos = 0 oldLoaded = 0 scanned = 0 existing = 0 duplicates = 0 ms = 0 assigned = 0 oldKept = 0 }
42. // the file of old ids: TRACKER_OLD_SHORT_IDS, else data/old-short-ids.json beside project.hl (relative paths resolve against the
43. // ENTRY script: a tool in tools/ — sync-tmdb.hl creates people since #28, mission 062 — finds it one folder up)
44. static oldFile = () => {
45. f = env('TRACKER_OLD_SHORT_IDS')
46. if (f != null && f.trim() != '') { return f.trim() }
47. return !exists('./data/old-short-ids.json') && exists('../data/old-short-ids.json') ? '../data/old-short-ids.json' : './data/old-short-ids.json'
48. }
50. // is this a short id at all (5 characters, a–z, 0–9)?
51. static isShortId = (&s) => {
52. if (s == null || hlTypeName(s) != 'String' || s.length != 5) { return false }
53. let i = 0
54. while (i < 5) {
55. if (!shortIdChars.includes(s[i])) { return false }
56. i = i + 1
57. }
58. return true
59. }
61. // 5 random characters a–z0–9, not taken and not an old one. Bytes ≥ 252 are skipped (252 = 7 × 36), so every character
62. // is equally likely.
63. static randomShortId = () => {
64. while (true) {
65. let hex = randomBytes(16, 'hex')
66. let out = ''
67. let i = 0
68. while (i < hex.length && out.length < 5) {
69. let b = hexDigits.indexOf(hex[i]) * 16 + hexDigits.indexOf(hex[i + 1])
70. if (b < 252) { out = out + shortIdChars[b % 36] }
71. i = i + 2
72. }
73. if (out.length == 5 && owners[out] == null && reserved[out] != true) { return out }
74. }
75. }
77. // the short id for a record that has none: its old one when the old tracker had one (and nobody holds it), else a new
78. // one — taken in `owners` at once
79. static claimShortId = (kind, id, &oldId) => {
80. ensureShortIds()
81. let sid = oldId != null ? oldShortIds['' + oldId] : null
82. if (sid != null && owners[sid] == null) { state.oldKept = state.oldKept + 1 } else { sid = randomShortId() }
83. owners[sid] = { kind = kind id = id }
84. return sid
85. }
87. // tracker#31 (merge.hl picks the keeper of duplicates): is this record's short id the one the old tracker gave it?
88. static hasOldShortId = (&r) => {
89. ensureShortIds()
90. return r.oldId != null && r.shortId != null && oldShortIds['' + r.oldId] == r.shortId
91. }
93. // does this record own its short id? (null / malformed / held by another record → no)
94. static ownsShortId = (&kind, &r) => {
95. if (!isShortId(r.shortId)) { return false }
96. o = owners[r.shortId]
97. return o != null && o.kind == kind && o.id == r.id
98. }
100. // one table's rows into `owners`; the rows without a (valid, own) short id into the queue
101. static scanTable = (kind, &table) => {
102. for (r of table.find(null, null)) {
103. state.scanned = state.scanned + 1
104. if (isShortId(r.shortId) && owners[r.shortId] == null) {
105. owners[r.shortId] = { kind = kind id = r.id }
106. state.existing = state.existing + 1
107. } else {
108. if (r.shortId != null) {
109. state.duplicates = state.duplicates + 1
110. console.log('short ids: ' + kind + ' ' + r.id + ' has "' + r.shortId + '", ' + (isShortId(r.shortId) ? 'taken by ' + owners[r.shortId].kind + ' ' + owners[r.shortId].id : 'not a short id') + ' — gets a new one')
111. }
112. queue.push({ kind = kind id = r.id })
113. }
114. }
115. return true
116. }
118. // at start (project.hl): the old ids, the map of every short id in use, the queue of records without one
119. static ensureShortIds = () => {
120. if (state.built) { return true }
121. state.built = true
122. t0 = now()
123. f = oldFile()
124. if (exists(f)) {
125. m = JSON.parse(readFile(f))
126. let n = 0
127. for (k of m.keys()) {
128. let sid = m[k]
129. if (isShortId(sid)) { oldShortIds[k] = sid reserved[sid] = true n = n + 1 }
130. }
131. state.oldLoaded = n
132. } else { console.log('short ids: no old ids (' + f + ' missing)') }
133. scanTable('show', showsTable)
134. scanTable('person', personsTable)
135. state.ms = now() - t0
136. console.log('short ids: ' + state.scanned + ' records, ' + state.existing + ' with a short id, ' + queue.length + ' without, ' + state.oldLoaded + ' old ids, ' + state.ms + ' ms')
137. return true
138. }
140. static tableOf = (&kind) => { return kind == 'person' ? personsTable : showsTable }
142. // the backfill, one batch: records from the queue until `budgetMs` is used up → { done, assigned, left }
143. static shortIdStep = (&budgetMs) => {
144. ensureShortIds()
145. t0 = now()
146. let n = 0
147. while (state.queuePos < queue.length && now() - t0 < budgetMs) {
148. let item = queue[state.queuePos]
149. state.queuePos = state.queuePos + 1
150. let table = tableOf(item.kind)
151. let r = table.fetch(item.id)
152. if (r != null && !ownsShortId(item.kind, r)) {
153. let x = r
154. x.shortId = claimShortId(item.kind, x.id, x.oldId)
155. if (table.update(x.id, x) == null) { console.log('short ids: ' + item.kind + ' ' + x.id + ' not stored: ' + table.lastError()) } else { n = n + 1 }
156. }
157. }
158. state.assigned = state.assigned + n
159. done = state.queuePos >= queue.length
160. return { done = done assigned = n left = queue.length - state.queuePos }
161. }
163. static shortIdStats = () => { return state }
164. static persistShortIds = () => {
165. showsTable.persist()
166. personsTable.persist()
167. return true
168. }
169. static shortIdsLeft = () => { return queue.length - state.queuePos }
171. // `/<shortId>` (project.hl): the page's path, or null
172. static shortIdPath = (&sid) => {
173. if (!isShortId(sid)) { return null }
174. ensureShortIds()
175. o = owners[sid]
176. if (o == null) { return null }
177. // tracker#31: a merged tombstone's short id leads to its keeper
178. r = o.kind == 'person' ? personsTable.fetch(o.id) : liveShowOf(showsTable.fetch(o.id))
179. if (r == null || r.urlSegment == null) { return null }
180. // a title's own page (/movies/, /series/, /shows/<slug>, tracker.worldapi.org#21)
181. return o.kind == 'person' ? '/person/' + r.urlSegment : titlePath(r)
182. }