tracker

18.0 KB

1. // tools/migrate.hl — ONE-OFF MIGRATION (tracker.worldapi.org#2): the old MongoDB export
2. // (/media/STORAGE/projects/old-tracker/mongo-export/*.jsonl, EJSON, one document per line)
3. // into this app's own mpackdb tables (storage/mpackdb/). EVERY record gets a NEW mpackdb id
4. // (self-assigned below — see "why self-assigned ids" — not mpackdb's auto-generated one); the
5. // old Mongo hex id is kept as `oldId` (a unique index: the idempotency key — a second run
6. // finds the same id again via `oldId` and skips the row, printing "already there").
7. //
8. // genres.db { name, tmdbId, urlSegment, oldId }
9. // persons.db { name, tmdbId, gender, urlSegment, birthDate, birthPlace, deathDate, image,
10. // imdbId, shows [{ show, character }], oldId }
11. // shows.db { title, summary, tmdbSummary, tmdbId, language, type, release, year, image,
12. // urlSegment, episodesCount, homepage, imdbId, seasonsCount, status, tagline,
13. // tvdbId, tvmzId, genres [genre id], cast [{ person, character, actor }],
14. // seasons [season id], oldId }
15. // seasons.db { seasonNumber, release, title, summary, show, episodes [episode id], oldId }
16. // episodes.db { tmdbId, episodeNumber, seasonNumber, title, summary, release, show, season,
17. // oldId } -- season is null where the OLD record's own season link was
18. // already null or dangling (pre-existing in the export, not introduced here)
19. // follows.db { show, user, at, oldId }
20. // watches.db { targetType ('episode'|'season'), target, user, at, oldId } -- target is
21. // null for the handful of watches whose old target no longer exists in the
22. // export (pre-existing dangling refs; kept so counts still match)
23. //
24. // The single old user (User.jsonl, [email protected], password NOT taken over) becomes
25. // this app's own user for ident short id az5b2 (users.hl `ensureUser`, storage/mpackdb/users.db
26. // — the SAME table step 1's login already uses).
27. //
28. // WHY SELF-ASSIGNED IDS, NO update(): Show.seasons/Season.show and Show.cast/Person.shows are
29. // each a genuine two-way reference (a show needs its seasons' ids; a season needs its show's id
30. // — same for shows and persons via cast). The first version of this tool broke that circle the
31. // obvious way — put() a bare record, come back later with fetch()+update() once the other side's
32. // ids exist. That LOSES ROWS: reopening a table in a fresh process after a run that had done
33. // update() on it read back fewer rows than were put (seen directly: count() dropped from 15157 to
34. // 15152 after persons.db round-tripped through update() once — a genuine hl:mpackdb bug, filed as
35. // a Hybriel gap, tracker.worldapi.org#2 — not something a vendored-plugin patch is allowed to fix
36. // here). Tables that only ever got put() never lost a row in the same testing.
37. // The fix: assign every new id ourselves (`resolveId`, 8 random bytes as hex, hl:crypto) in one
38. // pass over each file BEFORE building any record, so by the time a record is actually built every
39. // side of every reference is already a known id — one put() per row, with its id and every
40. // reference already resolved. mpackdb's own auto id is never used; hybriel#23 already establishes
41. // that put() keeps an id the record carries, which this leans on deliberately.
42. //
43. // Run (absolute paths; relative ones resolve against tools/, not the project root):
44. // TRACKER_OLD_DATA=/media/STORAGE/projects/old-tracker/mongo-export \
45. // TRACKER_STORAGE=$PWD/storage/mpackdb \
46. // TRACKER_MIGRATE_TMP=$PWD/.scratch/migrate-tmp \
47. // ./bin/hybriel tools/migrate.hl
48. import { MPackDB } from 'hl:mpackdb'
49. import { env, run } from 'hl:proc'
50. import { readFile, mkDir, exists } from 'hl:fs'
51. import { randomBytes } from 'hl:crypto'
52. import { ensureUser } from '../users.hl'
54. static nl = "
55. "
57. oldDir = env('TRACKER_OLD_DATA')
58. storageDir = env('TRACKER_STORAGE')
59. tmpDir = env('TRACKER_MIGRATE_TMP')
60. if (oldDir == null || !oldDir.startsWith('/') || storageDir == null || !storageDir.startsWith('/') || tmpDir == null || !tmpDir.startsWith('/')) {
61. hlError('set TRACKER_OLD_DATA (the mongo-export directory), TRACKER_STORAGE (the mpackdb directory) and TRACKER_MIGRATE_TMP (a scratch directory for the chunk workaround below), all ABSOLUTE')
62. }
64. // ---- reading the export --------------------------------------------------------------
65. static linesOf = (&path) => {
66. let raw = readFile(path)
67. if (raw == null) { return [] }
68. let out = []
69. for (l of raw.split(nl)) { if (l != '') { out.push(l) } }
70. return out
71. }
72. static oidOf = (&v) => {
73. if (v == null) { return null }
74. if (hlTypeName(v) == 'String') { return v }
75. return v['$oid']
76. }
77. static numOf = (&v) => {
78. if (v == null) { return null }
79. if (hlTypeName(v) == 'Number') { return v }
80. let a = v['$numberInt']
81. if (a == null) { a = v['$numberLong'] }
82. return a == null ? null : toNumber(a)
83. }
84. static dateOf = (&v) => {
85. if (v == null) { return null }
86. let d = v['$date']
87. return d == null ? null : numOf(d)
88. }
89. // a JSON list field that may be absent or explicitly null (most Show.cast/.seasons and
90. // nearly all Person.shows are) — iterating `null` with for-of aborts the whole run
91. static listOr = (&v) => { return v == null ? [] : v }
93. // hl:fs readFile SILENTLY TRUNCATES at 10 MiB (fs.zig hl_fs_read_file, max_size = 1024*1024*10,
94. // no error, no flag — reported as a Hybriel gap, tracker.worldapi.org#2): Show.jsonl (12.4 MB),
95. // Season.jsonl (10.9 MB) and Episode.jsonl (140 MB) are all over it. Worked around here WITHOUT
96. // touching the vendored plugin: `split` (coreutils, via hl:proc run — no shell, no JS) cuts the
97. // export into <10 MB chunks in a scratch dir first; each chunk is read whole (well under the cap).
98. static padNum4 = (&n) => {
99. let s = '' + n
100. while (s.length < 4) { s = '0' + s }
101. return s
102. }
103. static bigLinesOf = (&path, &chunkDir, &perChunk) => {
104. mkDir(chunkDir, 448)
105. let r = run(['split', '-l', '' + perChunk, '-d', '-a', '4', '--additional-suffix=.jsonl', path, chunkDir + '/c_'])
106. if (r == null || r.exit != 0) { hlError('split failed for ' + path) }
107. let out = []
108. let idx = 0
109. while (true) {
110. let fp = chunkDir + '/c_' + padNum4(idx) + '.jsonl'
111. if (!exists(fp)) { break }
112. for (l of linesOf(fp)) { out.push(l) }
113. idx = idx + 1
114. }
115. if (idx == 0) { hlError('no chunks produced for ' + path) }
116. return out
117. }
119. // ---- tables ---------------------------------------------------------------------------
120. genresTable = new MPackDB(file = storageDir + '/genres.db', primaryKey = '@id', indexes = ['!oldId'])
121. personsTable = new MPackDB(file = storageDir + '/persons.db', primaryKey = '@id', indexes = ['!oldId'])
122. showsTable = new MPackDB(file = storageDir + '/shows.db', primaryKey = '@id', indexes = ['!oldId'])
123. seasonsTable = new MPackDB(file = storageDir + '/seasons.db', primaryKey = '@id', indexes = ['!oldId'])
124. episodesTable = new MPackDB(file = storageDir + '/episodes.db', primaryKey = '@id', indexes = ['!oldId'])
125. followsTable = new MPackDB(file = storageDir + '/follows.db', primaryKey = '@id', indexes = ['!oldId'])
126. watchesTable = new MPackDB(file = storageDir + '/watches.db', primaryKey = '@id', indexes = ['!oldId'])
128. // the new id for an old row: reuse what a previous run already gave it (a point lookup on the
129. // unique `oldId` index — proven reliable; it's a FULL table scan, find(null, null), that lost
130. // rows together with update() in testing, so this never calls that), or a fresh random id
131. static newId = () => { return randomBytes(8, 'hex') }
132. static existingId = (&table, &oldId) => {
133. let found = table.find('oldId', oldId)
134. if (found != null && found.length > 0) { return found[0].id }
135. return null
136. }
137. static resolveId = (&table, &oldId) => {
138. let ex = existingId(table, oldId)
139. return ex == null ? newId() : ex
140. }
142. report = []
143. failed = 0
145. // ---- read every file once (chunked for the big three) --------------------------------
146. genreLines = linesOf(oldDir + '/Genre.jsonl')
147. personLines = linesOf(oldDir + '/Person.jsonl')
148. showLines = bigLinesOf(oldDir + '/Show.jsonl', tmpDir + '/show', 5000)
149. seasonLines = bigLinesOf(oldDir + '/Season.jsonl', tmpDir + '/season', 3000)
150. episodeLines = bigLinesOf(oldDir + '/Episode.jsonl', tmpDir + '/episode', 5000)
151. followLines = linesOf(oldDir + '/Follow.jsonl')
152. watchLines = linesOf(oldDir + '/Watch.jsonl')
154. // ---- phase 1: id maps only (every old id -> its new id, self-assigned) ---------------
155. // every reference below (Show.genres/cast/seasons, Season.show/episodes, Episode.show/season,
156. // Person.shows, Follow.oid, Watch.oid) is resolved against one of these five maps, ALL of
157. // which are complete before phase 2 builds a single record — that is what removes the need
158. // for update() (see the file header).
159. genreMap = {}
160. for (l of genreLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) genreMap[oldId] = resolveId(genresTable, oldId) }
161. personMap = {}
162. for (l of personLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) personMap[oldId] = resolveId(personsTable, oldId) }
163. showMap = {}
164. for (l of showLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) showMap[oldId] = resolveId(showsTable, oldId) }
165. seasonMap = {}
166. for (l of seasonLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) seasonMap[oldId] = resolveId(seasonsTable, oldId) }
167. episodeMap = {}
168. for (l of episodeLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) episodeMap[oldId] = resolveId(episodesTable, oldId) }
170. // ---- phase 2: build and put the real records (one put() each, every reference resolved) ----
172. // ---- Genre ----------------------------------------------------------------------------
173. genreFresh = 0
174. genreKnown = 0
175. for (l of genreLines) {
176. let d = JSON.parse(l)
177. let oldId = oidOf(d._id)
178. let id = genreMap[oldId]
179. if (genresTable.fetch(id) != null) {
180. genreKnown = genreKnown + 1
181. } else {
182. let putId = genresTable.put({ id = id name = d.name tmdbId = numOf(d.tmdbId) urlSegment = d.urlSegment oldId = oldId })
183. if (putId == null) { console.log('FAIL genre ' + oldId + ': ' + genresTable.lastError()) failed = failed + 1 } else { genreFresh = genreFresh + 1 }
184. }
185. }
186. report.push('genres ' + genreFresh + ' new, ' + genreKnown + ' already there (old ' + genreLines.length + ' -> new ' + genresTable.count() + ')')
188. // ---- Person (shows resolved against showMap, already complete) -----------------------
189. personFresh = 0
190. personKnown = 0
191. personShowsMissing = 0
192. for (l of personLines) {
193. let d = JSON.parse(l)
194. let oldId = oidOf(d._id)
195. let id = personMap[oldId]
196. if (personsTable.fetch(id) != null) {
197. personKnown = personKnown + 1
198. } else {
199. let shows = []
200. for (s of listOr(d.shows)) {
201. let shid = showMap[oidOf(s._id)]
202. if (shid == null) { personShowsMissing = personShowsMissing + 1 } else { shows.push({ show = shid character = s.character }) }
203. }
204. let putId = personsTable.put({
205. id = id name = d.name tmdbId = numOf(d.tmdbId) gender = numOf(d.gender) urlSegment = d.urlSegment
206. birthDate = d.birthDate birthPlace = d.birthPlace deathDate = d.deathDate image = d.image
207. imdbId = d.imdbId shows = shows oldId = oldId
208. })
209. if (putId == null) { console.log('FAIL person ' + oldId + ': ' + personsTable.lastError()) failed = failed + 1 } else { personFresh = personFresh + 1 }
210. }
211. }
212. report.push('persons ' + personFresh + ' new, ' + personKnown + ' already there (old ' + personLines.length + ' -> new ' + personsTable.count() + '), show refs skipped ' + personShowsMissing)
214. // ---- Show (genres/cast/seasons resolved against genreMap/personMap/seasonMap) --------
215. showFresh = 0
216. showKnown = 0
217. castMissing = 0
218. showGenresMissing = 0
219. showSeasonsMissing = 0
220. for (l of showLines) {
221. let d = JSON.parse(l)
222. let oldId = oidOf(d._id)
223. let id = showMap[oldId]
224. if (showsTable.fetch(id) != null) {
225. showKnown = showKnown + 1
226. } else {
227. let genres = []
228. for (g of listOr(d.genres)) {
229. let gid = genreMap[oidOf(g._id)]
230. if (gid == null) { showGenresMissing = showGenresMissing + 1 } else { genres.push(gid) }
231. }
232. let cast = []
233. for (c of listOr(d.cast)) {
234. let pid = personMap[oidOf(c._id)]
235. if (pid == null) { castMissing = castMissing + 1 } else { cast.push({ person = pid character = c.character actor = c.actor }) }
236. }
237. let seasons = []
238. for (s of listOr(d.seasons)) {
239. let sid = seasonMap[oidOf(s)]
240. if (sid == null) { showSeasonsMissing = showSeasonsMissing + 1 } else { seasons.push(sid) }
241. }
242. let putId = showsTable.put({
243. id = id title = d.title summary = d.summary tmdbSummary = d.tmdbSummary tmdbId = numOf(d.tmdbId)
244. language = d.language type = d.type release = d.release year = numOf(d.year) image = d.image
245. urlSegment = d.urlSegment episodesCount = numOf(d.episodesCount) homepage = d.homepage
246. imdbId = d.imdbId seasonsCount = numOf(d.seasonsCount) status = d.status tagline = d.tagline
247. tvdbId = numOf(d.tvdbId) tvmzId = numOf(d.tvmzId) genres = genres cast = cast seasons = seasons
248. oldId = oldId
249. })
250. if (putId == null) { console.log('FAIL show ' + oldId + ': ' + showsTable.lastError()) failed = failed + 1 } else { showFresh = showFresh + 1 }
251. }
252. }
253. report.push('shows ' + showFresh + ' new, ' + showKnown + ' already there (old ' + showLines.length + ' -> new ' + showsTable.count() + '), cast refs skipped ' + castMissing + ', genre refs skipped ' + showGenresMissing + ', season refs skipped ' + showSeasonsMissing)
255. // ---- Season (show resolved against showMap, episodes against episodeMap) -------------
256. seasonFresh = 0
257. seasonKnown = 0
258. seasonShowMissing = 0
259. seasonEpisodesMissing = 0
260. for (l of seasonLines) {
261. let d = JSON.parse(l)
262. let oldId = oidOf(d._id)
263. let id = seasonMap[oldId]
264. if (seasonsTable.fetch(id) != null) {
265. seasonKnown = seasonKnown + 1
266. } else {
267. let show = showMap[oidOf(d.show._id)]
268. if (show == null) { seasonShowMissing = seasonShowMissing + 1 }
269. let episodes = []
270. for (e of listOr(d.episodes)) {
271. let eid = episodeMap[oidOf(e)]
272. if (eid == null) { seasonEpisodesMissing = seasonEpisodesMissing + 1 } else { episodes.push(eid) }
273. }
274. let putId = seasonsTable.put({ id = id seasonNumber = numOf(d.seasonNumber) release = d.release title = d.title summary = d.summary show = show episodes = episodes oldId = oldId })
275. if (putId == null) { console.log('FAIL season ' + oldId + ': ' + seasonsTable.lastError()) failed = failed + 1 } else { seasonFresh = seasonFresh + 1 }
276. }
277. }
278. report.push('seasons ' + seasonFresh + ' new, ' + seasonKnown + ' already there (old ' + seasonLines.length + ' -> new ' + seasonsTable.count() + '), show refs skipped ' + seasonShowMissing + ', episode refs skipped ' + seasonEpisodesMissing)
280. // ---- Episode (show/season resolved; season null where the OLD link was already null or
281. // dangling — pre-existing in the export, not introduced here) -------------------------
282. episodeFresh = 0
283. episodeKnown = 0
284. episodeShowMissing = 0
285. episodeSeasonMissing = 0
286. for (l of episodeLines) {
287. let d = JSON.parse(l)
288. let oldId = oidOf(d._id)
289. let id = episodeMap[oldId]
290. if (episodesTable.fetch(id) != null) {
291. episodeKnown = episodeKnown + 1
292. } else {
293. let show = showMap[oidOf(d.show._id)]
294. if (show == null) { episodeShowMissing = episodeShowMissing + 1 }
295. let oldSeasonId = oidOf(d.season._id)
296. let season = oldSeasonId == null ? null : seasonMap[oldSeasonId]
297. if (oldSeasonId != null && season == null) { episodeSeasonMissing = episodeSeasonMissing + 1 }
298. let putId = episodesTable.put({
299. id = id tmdbId = numOf(d.tmdbId) episodeNumber = numOf(d.episodeNumber) seasonNumber = numOf(d.seasonNumber)
300. title = d.title summary = d.summary release = d.release show = show season = season oldId = oldId
301. })
302. if (putId == null) { console.log('FAIL episode ' + oldId + ': ' + episodesTable.lastError()) failed = failed + 1 } else { episodeFresh = episodeFresh + 1 }
303. }
304. }
305. report.push('episodes ' + episodeFresh + ' new, ' + episodeKnown + ' already there (old ' + episodeLines.length + ' -> new ' + episodesTable.count() + '), show refs skipped ' + episodeShowMissing + ', season refs left null (pre-existing) ' + episodeSeasonMissing)
307. // ---- the single old user -> this app's user for ident short id az5b2 ---------------
308. creator = ensureUser('az5b2')
309. if (creator == null) { hlError('could not create/find the user for az5b2') }
310. userId = creator.id
312. // ---- Follow ---------------------------------------------------------------------
313. followFresh = 0
314. followKnown = 0
315. followMissing = 0
316. for (l of followLines) {
317. let d = JSON.parse(l)
318. let oldId = oidOf(d._id)
319. let id = resolveId(followsTable, oldId)
320. if (followsTable.fetch(id) != null) {
321. followKnown = followKnown + 1
322. } else {
323. let show = showMap[oidOf(d.oid)]
324. if (show == null) { followMissing = followMissing + 1 }
325. let putId = followsTable.put({ id = id show = show user = userId at = dateOf(d._createdAt) oldId = oldId })
326. if (putId == null) { console.log('FAIL follow ' + oldId + ': ' + followsTable.lastError()) failed = failed + 1 } else { followFresh = followFresh + 1 }
327. }
328. }
329. report.push('follows ' + followFresh + ' new, ' + followKnown + ' already there (old ' + followLines.length + ' -> new ' + followsTable.count() + '), show refs skipped ' + followMissing)
331. // ---- Watch (target null where the OLD target no longer exists — pre-existing) -----
332. watchFresh = 0
333. watchKnown = 0
334. watchMissing = 0
335. for (l of watchLines) {
336. let d = JSON.parse(l)
337. let oldId = oidOf(d._id)
338. let id = resolveId(watchesTable, oldId)
339. if (watchesTable.fetch(id) != null) {
340. watchKnown = watchKnown + 1
341. } else {
342. let kind = d.type
343. let oldTarget = oidOf(d.oid)
344. let target = kind == 'episode' ? episodeMap[oldTarget] : seasonMap[oldTarget]
345. if (target == null) { watchMissing = watchMissing + 1 }
346. let putId = watchesTable.put({ id = id targetType = kind target = target user = userId at = dateOf(d._createdAt) oldId = oldId })
347. if (putId == null) { console.log('FAIL watch ' + oldId + ': ' + watchesTable.lastError()) failed = failed + 1 } else { watchFresh = watchFresh + 1 }
348. }
349. }
350. report.push('watches ' + watchFresh + ' new, ' + watchKnown + ' already there (old ' + watchLines.length + ' -> new ' + watchesTable.count() + '), target refs left null (pre-existing) ' + watchMissing)
352. genresTable.persist()
353. personsTable.persist()
354. showsTable.persist()
355. seasonsTable.persist()
356. episodesTable.persist()
357. followsTable.persist()
358. watchesTable.persist()
360. for (line of report) { console.log('migrate: ' + line) }
361. console.log('migrate: ' + failed + ' failed (' + oldDir + ' -> ' + storageDir + ')')