tracker

8.1 KB

1. // shortids.hl — SHORT IDS (tracker.worldapi.org#27, mission 060): every movie, series and person has its own 5-character id
2. // (`shortId`, like `a9s9a`), shown below its poster/photo; `/<shortId>` answers 301 to its page. Creator: "i wanted to give all
3. // our movies, shows, series and persons own short ids like a9s9a … those ids show below the poster."
4. //
5. // OLD IDS: the old tracker had them on 702 of 9,453 shows (Show.jsonl field `id`, base62 counter '100jh'…'1012y', all of
6. // them [a-z0-9]{5}). `data/old-short-ids.json` (made once by tools/old-short-ids.hl) maps the old Mongo `_id` (= our
7. // `oldId`, tools/migrate.hl) → that id; such a show gets it back. Persons never had one.
8. // NEW IDS: 5 random characters [a-z0-9] (36^5 = 60 M) — never one that is taken or one of the old ids (all 702 are
9. // reserved, even where the show is gone).
10. // UNIQUE ACROSS SHOWS AND PERSONS: one in-memory map `owners` (short id → { kind, id }) over BOTH tables — an mpackdb index
11. // is per table and cannot span two. Built at start by `ensureShortIds` (one scan of each table); every id handed
12. // out goes through it. A record whose short id another record already owns (never seen; logged) gets a new one.
13. // NEW RECORDS get one when they are created: `claimShortId(kind, id, oldId)` in search.hl importTitle and people.hl
14. // addMinimalTitle (the only places the app creates shows) and people.hl castPersonId (mission 062: the only place
15. // the app creates persons — the cast/crew of #26/#28: completion, credits job, daily sync + tools/sync-tmdb.hl,
16. // search import; and a guest star opened for the first time, people.hl guestPersonId).
17. // THE BACKFILL: the records without one (at first every row: 9,453 shows + 15,157 persons) are given theirs in the
18. // background, a time-boxed batch per clock tick (project.hl `shortIdTick` → `shortIdStep`), so pages are served
19. // between two batches; after a restart it goes on with what is still missing. Real data: STATUS.md mission 060.
20. //
21. // ONE PROCESS PER STORAGE (hybriel#21): the map lives in the app's process; the tables are shows.hl's.
22. import { now } from 'hl:time'
23. import { env } from 'hl:proc'
24. import { readFile, exists } from 'hl:fs'
25. import { randomBytes } from 'hl:crypto'
26. import { showsTable, personsTable, titlePath } from './shows.hl'
28. static shortIdChars = 'abcdefghijklmnopqrstuvwxyz0123456789'
29. static hexDigits = '0123456789abcdef'
31. // short id → { kind ('show' | 'person'), id }
32. static owners = {}
33. // the old tracker's ids: oldId → short id, and every one of them → true (reserved)
34. static oldShortIds = {}
35. static reserved = {}
36. // the records still without a short id, [{ kind, id }], worked off from `state.queuePos`. Filled in place (push) and never
37. // reassigned (a static cannot be) — kept OUT of `state`: reading `state.queue[i]` copied the whole 25k list on every access
38. // (measured: ~3 ms per record instead of ~0.2)
39. static queue = []
40. static state = { built = false queuePos = 0 oldLoaded = 0 scanned = 0 existing = 0 duplicates = 0 ms = 0 assigned = 0 oldKept = 0 }
42. // the file of old ids: TRACKER_OLD_SHORT_IDS, else data/old-short-ids.json beside project.hl (relative paths resolve against the
43. // ENTRY script: a tool in tools/ — sync-tmdb.hl creates people since #28, mission 062 — finds it one folder up)
44. static oldFile = () => {
45. let f = env('TRACKER_OLD_SHORT_IDS')
46. if (f != null && f.trim() != '') { return f.trim() }
47. return !exists('./data/old-short-ids.json') && exists('../data/old-short-ids.json') ? '../data/old-short-ids.json' : './data/old-short-ids.json'
48. }
50. // is this a short id at all (5 characters, a–z, 0–9)?
51. static isShortId = (&s) => {
52. if (s == null || hlTypeName(s) != 'String' || s.length != 5) { return false }
53. let i = 0
54. while (i < 5) {
55. if (!shortIdChars.includes(s[i])) { return false }
56. i = i + 1
57. }
58. return true
59. }
61. // 5 random characters a–z0–9, not taken and not an old one. Bytes ≥ 252 are skipped (252 = 7 × 36), so every character
62. // is equally likely.
63. static randomShortId = () => {
64. while (true) {
65. let hex = randomBytes(16, 'hex')
66. let out = ''
67. let i = 0
68. while (i < hex.length && out.length < 5) {
69. let b = hexDigits.indexOf(hex[i]) * 16 + hexDigits.indexOf(hex[i + 1])
70. if (b < 252) { out = out + shortIdChars[b % 36] }
71. i = i + 2
72. }
73. if (out.length == 5 && owners[out] == null && reserved[out] != true) { return out }
74. }
75. }
77. // the short id for a record that has none: its old one when the old tracker had one (and nobody holds it), else a new
78. // one — taken in `owners` at once
79. static claimShortId = (kind, id, &oldId) => {
80. ensureShortIds()
81. let sid = oldId != null ? oldShortIds['' + oldId] : null
82. if (sid != null && owners[sid] == null) { state.oldKept = state.oldKept + 1 } else { sid = randomShortId() }
83. owners[sid] = { kind = kind id = id }
84. return sid
85. }
87. // does this record own its short id? (null / malformed / held by another record → no)
88. static ownsShortId = (&kind, &r) => {
89. if (!isShortId(r.shortId)) { return false }
90. let o = owners[r.shortId]
91. return o != null && o.kind == kind && o.id == r.id
92. }
94. // one table's rows into `owners`; the rows without a (valid, own) short id into the queue
95. static scanTable = (kind, &table) => {
96. for (r of table.find(null, null)) {
97. state.scanned = state.scanned + 1
98. if (isShortId(r.shortId) && owners[r.shortId] == null) {
99. owners[r.shortId] = { kind = kind id = r.id }
100. state.existing = state.existing + 1
101. } else {
102. if (r.shortId != null) {
103. state.duplicates = state.duplicates + 1
104. console.log('short ids: ' + kind + ' ' + r.id + ' has "' + r.shortId + '", ' + (isShortId(r.shortId) ? 'taken by ' + owners[r.shortId].kind + ' ' + owners[r.shortId].id : 'not a short id') + ' — gets a new one')
105. }
106. queue.push({ kind = kind id = r.id })
107. }
108. }
109. return true
110. }
112. // at start (project.hl): the old ids, the map of every short id in use, the queue of records without one
113. static ensureShortIds = () => {
114. if (state.built) { return true }
115. state.built = true
116. let t0 = now()
117. let f = oldFile()
118. if (exists(f)) {
119. let m = JSON.parse(readFile(f))
120. let n = 0
121. for (k of m.keys()) {
122. let sid = m[k]
123. if (isShortId(sid)) { oldShortIds[k] = sid reserved[sid] = true n = n + 1 }
124. }
125. state.oldLoaded = n
126. } else { console.log('short ids: no old ids (' + f + ' missing)') }
127. scanTable('show', showsTable)
128. scanTable('person', personsTable)
129. state.ms = now() - t0
130. console.log('short ids: ' + state.scanned + ' records, ' + state.existing + ' with a short id, ' + queue.length + ' without, ' + state.oldLoaded + ' old ids, ' + state.ms + ' ms')
131. return true
132. }
134. static tableOf = (&kind) => { return kind == 'person' ? personsTable : showsTable }
136. // the backfill, one batch: records from the queue until `budgetMs` is used up → { done, assigned, left }
137. static shortIdStep = (&budgetMs) => {
138. ensureShortIds()
139. let t0 = now()
140. let n = 0
141. while (state.queuePos < queue.length && now() - t0 < budgetMs) {
142. let item = queue[state.queuePos]
143. state.queuePos = state.queuePos + 1
144. let table = tableOf(item.kind)
145. let r = table.fetch(item.id)
146. if (r != null && !ownsShortId(item.kind, r)) {
147. let x = r
148. x.shortId = claimShortId(item.kind, x.id, x.oldId)
149. if (table.update(x.id, x) == null) { console.log('short ids: ' + item.kind + ' ' + x.id + ' not stored: ' + table.lastError()) } else { n = n + 1 }
150. }
151. }
152. state.assigned = state.assigned + n
153. let done = state.queuePos >= queue.length
154. return { done = done assigned = n left = queue.length - state.queuePos }
155. }
157. static shortIdStats = () => { return state }
158. static persistShortIds = () => {
159. showsTable.persist()
160. personsTable.persist()
161. return true
162. }
163. static shortIdsLeft = () => { return queue.length - state.queuePos }
165. // `/<shortId>` (project.hl): the page's path, or null
166. static shortIdPath = (&sid) => {
167. if (!isShortId(sid)) { return null }
168. ensureShortIds()
169. let o = owners[sid]
170. if (o == null) { return null }
171. let r = tableOf(o.kind).fetch(o.id)
172. if (r == null || r.urlSegment == null) { return null }
173. // a title's own page (/movies/, /series/, /shows/<slug>, tracker.worldapi.org#21)
174. return o.kind == 'person' ? '/person/' + r.urlSegment : titlePath(r)
175. }