tracker

7.5 KB

1. // shortids.hl — SHORT IDS (tracker.worldapi.org#27, mission 060): every movie, series and person has its own 5-character id
2. // (`shortId`, like `a9s9a`), shown below its poster/photo; `/<shortId>` answers 301 to its page. Creator: "i wanted to give all
3. // our movies, shows, series and persons own short ids like a9s9a … those ids show below the poster."
4. //
5. // OLD IDS: the old tracker had them on 702 of 9,453 shows (Show.jsonl field `id`, base62 counter '100jh'…'1012y', all of
6. // them [a-z0-9]{5}). `data/old-short-ids.json` (made once by tools/old-short-ids.hl) maps the old Mongo `_id` (= our
7. // `oldId`, tools/migrate.hl) → that id; such a show gets it back. Persons never had one.
8. // NEW IDS: 5 random characters [a-z0-9] (36^5 = 60 M) — never one that is taken or one of the old ids (all 702 are
9. // reserved, even where the show is gone).
10. // UNIQUE ACROSS SHOWS AND PERSONS: one in-memory map `owners` (short id → { kind, id }) over BOTH tables — an mpackdb index
11. // is per table and cannot span two. Built at start by `ensureShortIds` (one scan of each table); every id handed
12. // out goes through it. A record whose short id another record already owns (never seen; logged) gets a new one.
13. // NEW RECORDS get one when they are created: `claimShortId(kind, id, oldId)` in search.hl importTitle and people.hl
14. // addMinimalTitle (the only places the app creates shows; persons are only created by tools/migrate.hl).
15. // THE BACKFILL: the records without one (at first every row: 9,453 shows + 15,157 persons) are given theirs in the
16. // background, a time-boxed batch per clock tick (project.hl `shortIdTick` → `shortIdStep`), so pages are served
17. // between two batches; after a restart it goes on with what is still missing. Real data: STATUS.md mission 060.
18. //
19. // ONE PROCESS PER STORAGE (hybriel#21): the map lives in the app's process; the tables are shows.hl's.
20. import { now } from 'hl:time'
21. import { env } from 'hl:proc'
22. import { readFile, exists } from 'hl:fs'
23. import { randomBytes } from 'hl:crypto'
24. import { showsTable, personsTable } from './shows.hl'
26. static shortIdChars = 'abcdefghijklmnopqrstuvwxyz0123456789'
27. static hexDigits = '0123456789abcdef'
29. // short id → { kind ('show' | 'person'), id }
30. static owners = {}
31. // the old tracker's ids: oldId → short id, and every one of them → true (reserved)
32. static oldShortIds = {}
33. static reserved = {}
34. // the records still without a short id, [{ kind, id }], worked off from `state.queuePos`. Filled in place (push) and never
35. // reassigned (a static cannot be) — kept OUT of `state`: reading `state.queue[i]` copied the whole 25k list on every access
36. // (measured: ~3 ms per record instead of ~0.2)
37. static queue = []
38. static state = { built = false queuePos = 0 oldLoaded = 0 scanned = 0 existing = 0 duplicates = 0 ms = 0 assigned = 0 oldKept = 0 }
40. // the file of old ids: TRACKER_OLD_SHORT_IDS, else data/old-short-ids.json beside project.hl
41. static oldFile = () => {
42. let f = env('TRACKER_OLD_SHORT_IDS')
43. return f != null && f.trim() != '' ? f.trim() : './data/old-short-ids.json'
44. }
46. // is this a short id at all (5 characters, a–z, 0–9)?
47. static isShortId = (s) => {
48. if (s == null || hlTypeName(s) != 'String' || s.length != 5) { return false }
49. let i = 0
50. while (i < 5) {
51. if (!shortIdChars.includes(s[i])) { return false }
52. i = i + 1
53. }
54. return true
55. }
57. // 5 random characters a–z0–9, not taken and not an old one. Bytes ≥ 252 are skipped (252 = 7 × 36), so every character
58. // is equally likely.
59. static randomShortId = () => {
60. while (true) {
61. let hex = randomBytes(16, 'hex')
62. let out = ''
63. let i = 0
64. while (i < hex.length && out.length < 5) {
65. let b = hexDigits.indexOf(hex[i]) * 16 + hexDigits.indexOf(hex[i + 1])
66. if (b < 252) { out = out + shortIdChars[b % 36] }
67. i = i + 2
68. }
69. if (out.length == 5 && owners[out] == null && reserved[out] != true) { return out }
70. }
71. }
73. // the short id for a record that has none: its old one when the old tracker had one (and nobody holds it), else a new
74. // one — taken in `owners` at once
75. static claimShortId = (kind, id, oldId) => {
76. ensureShortIds()
77. let sid = oldId != null ? oldShortIds['' + oldId] : null
78. if (sid != null && owners[sid] == null) { state.oldKept = state.oldKept + 1 } else { sid = randomShortId() }
79. owners[sid] = { kind = kind id = id }
80. return sid
81. }
83. // does this record own its short id? (null / malformed / held by another record → no)
84. static ownsShortId = (kind, r) => {
85. if (!isShortId(r.shortId)) { return false }
86. let o = owners[r.shortId]
87. return o != null && o.kind == kind && o.id == r.id
88. }
90. // one table's rows into `owners`; the rows without a (valid, own) short id into the queue
91. static scanTable = (kind, table) => {
92. for (r of table.find(null, null)) {
93. state.scanned = state.scanned + 1
94. if (isShortId(r.shortId) && owners[r.shortId] == null) {
95. owners[r.shortId] = { kind = kind id = r.id }
96. state.existing = state.existing + 1
97. } else {
98. if (r.shortId != null) {
99. state.duplicates = state.duplicates + 1
100. console.log('short ids: ' + kind + ' ' + r.id + ' has "' + r.shortId + '", ' + (isShortId(r.shortId) ? 'taken by ' + owners[r.shortId].kind + ' ' + owners[r.shortId].id : 'not a short id') + ' — gets a new one')
101. }
102. queue.push({ kind = kind id = r.id })
103. }
104. }
105. return true
106. }
108. // at start (project.hl): the old ids, the map of every short id in use, the queue of records without one
109. static ensureShortIds = () => {
110. if (state.built) { return true }
111. state.built = true
112. let t0 = now()
113. let f = oldFile()
114. if (exists(f)) {
115. let m = JSON.parse(readFile(f))
116. let n = 0
117. for (k of m.keys()) {
118. let sid = m[k]
119. if (isShortId(sid)) { oldShortIds[k] = sid reserved[sid] = true n = n + 1 }
120. }
121. state.oldLoaded = n
122. } else { console.log('short ids: no old ids (' + f + ' missing)') }
123. scanTable('show', showsTable)
124. scanTable('person', personsTable)
125. state.ms = now() - t0
126. console.log('short ids: ' + state.scanned + ' records, ' + state.existing + ' with a short id, ' + queue.length + ' without, ' + state.oldLoaded + ' old ids, ' + state.ms + ' ms')
127. return true
128. }
130. static tableOf = (kind) => { return kind == 'person' ? personsTable : showsTable }
132. // the backfill, one batch: records from the queue until `budgetMs` is used up → { done, assigned, left }
133. static shortIdStep = (budgetMs) => {
134. ensureShortIds()
135. let t0 = now()
136. let n = 0
137. while (state.queuePos < queue.length && now() - t0 < budgetMs) {
138. let item = queue[state.queuePos]
139. state.queuePos = state.queuePos + 1
140. let table = tableOf(item.kind)
141. let r = table.fetch(item.id)
142. if (r != null && !ownsShortId(item.kind, r)) {
143. let x = r
144. x.shortId = claimShortId(item.kind, x.id, x.oldId)
145. if (table.update(x.id, x) == null) { console.log('short ids: ' + item.kind + ' ' + x.id + ' not stored: ' + table.lastError()) } else { n = n + 1 }
146. }
147. }
148. state.assigned = state.assigned + n
149. let done = state.queuePos >= queue.length
150. return { done = done assigned = n left = queue.length - state.queuePos }
151. }
153. static shortIdStats = () => { return state }
154. static persistShortIds = () => {
155. showsTable.persist()
156. personsTable.persist()
157. return true
158. }
159. static shortIdsLeft = () => { return queue.length - state.queuePos }
161. // `/<shortId>` (project.hl): the page's path, or null
162. static shortIdPath = (sid) => {
163. if (!isShortId(sid)) { return null }
164. ensureShortIds()
165. let o = owners[sid]
166. if (o == null) { return null }
167. let r = tableOf(o.kind).fetch(o.id)
168. if (r == null || r.urlSegment == null) { return null }
169. return (o.kind == 'person' ? '/person/' : '/shows/') + r.urlSegment
170. }