tracker

16.6 KB

1. // tools/migrate.hl — ONE-OFF MIGRATION (tracker.worldapi.org#2): the old MongoDB export
2. // (/media/STORAGE/projects/old-tracker/mongo-export/*.jsonl, EJSON, one document per line)
3. // into this app's own mpackdb tables (storage/mpackdb/). EVERY record gets a NEW mpackdb id
4. // (self-assigned below — see "why self-assigned ids" — not mpackdb's auto-generated one); the
5. // old Mongo hex id is kept as `oldId` (a unique index: the idempotency key — a second run
6. // finds the same id again via `oldId` and skips the row, printing "already there").
7. //
8. // genres.db { name, tmdbId, urlSegment, oldId }
9. // persons.db { name, tmdbId, gender, urlSegment, birthDate, birthPlace, deathDate, image,
10. // imdbId, shows [{ show, character }], oldId }
11. // shows.db { title, summary, tmdbSummary, tmdbId, language, type, release, year, image,
12. // urlSegment, episodesCount, homepage, imdbId, seasonsCount, status, tagline,
13. // tvdbId, tvmzId, genres [genre id], cast [{ person, character, actor }],
14. // seasons [season id], oldId }
15. // seasons.db { seasonNumber, release, title, summary, show, episodes [episode id], oldId }
16. // episodes.db { tmdbId, episodeNumber, seasonNumber, title, summary, release, show, season,
17. // oldId } -- season is null where the OLD record's own season link was
18. // already null or dangling (pre-existing in the export, not introduced here)
19. // follows.db { show, user, at, oldId }
20. // watches.db { targetType ('episode'|'season'), target, user, at, oldId } -- target is
21. // null for the handful of watches whose old target no longer exists in the
22. // export (pre-existing dangling refs; kept so counts still match)
23. //
24. // The single old user (User.jsonl, [email protected], password NOT taken over) becomes
25. // this app's own user for ident short id az5b2 (users.hl `ensureUser`, storage/mpackdb/users.db
26. // — the SAME table step 1's login already uses).
27. //
28. // WHY SELF-ASSIGNED IDS, NO update(): Show.seasons/Season.show and Show.cast/Person.shows are
29. // each a genuine two-way reference (a show needs its seasons' ids; a season needs its show's id
30. // — same for shows and persons via cast). The first version of this tool broke that circle the
31. // obvious way — put() a bare record, come back later with fetch()+update() once the other side's
32. // ids exist. That LOSES ROWS: reopening a table in a fresh process after a run that had done
33. // update() on it read back fewer rows than were put (seen directly: count() dropped from 15157 to
34. // 15152 after persons.db round-tripped through update() once — a genuine hl:mpackdb bug, filed as
35. // a Hybriel gap, tracker.worldapi.org#2 — not something a vendored-plugin patch is allowed to fix
36. // here). Tables that only ever got put() never lost a row in the same testing.
37. // The fix: assign every new id ourselves (`resolveId`, 8 random bytes as hex, hl:crypto) in one
38. // pass over each file BEFORE building any record, so by the time a record is actually built every
39. // side of every reference is already a known id — one put() per row, with its id and every
40. // reference already resolved. mpackdb's own auto id is never used; hybriel#23 already establishes
41. // that put() keeps an id the record carries, which this leans on deliberately.
42. //
43. // Run (absolute paths; relative ones resolve against tools/, not the project root):
44. // TRACKER_OLD_DATA=/media/STORAGE/projects/old-tracker/mongo-export \
45. // TRACKER_STORAGE=$PWD/storage/mpackdb \
46. // TRACKER_MIGRATE_TMP=$PWD/.scratch/migrate-tmp \
47. // ./bin/hybriel tools/migrate.hl
48. import { MPackDB } from 'hl:mpackdb'
49. import { env } from 'hl:proc'
50. import { newId, listOr } from '../lib/util.hl'
51. import { linesOf, oidOf, bigLinesOf } from '../lib/export.hl'
52. import { ensureUser } from '../lib/users.hl'
54. oldDir = env('TRACKER_OLD_DATA')
55. storageDir = env('TRACKER_STORAGE')
56. tmpDir = env('TRACKER_MIGRATE_TMP')
57. if (oldDir == null || !oldDir.startsWith('/') || storageDir == null || !storageDir.startsWith('/') || tmpDir == null || !tmpDir.startsWith('/')) {
58. hlError('set TRACKER_OLD_DATA (the mongo-export directory), TRACKER_STORAGE (the mpackdb directory) and TRACKER_MIGRATE_TMP (a scratch directory for the chunk workaround below), all ABSOLUTE')
59. }
61. // ---- reading the export (lib/export.hl: linesOf, oidOf, bigLinesOf) ------------------------------
62. static numOf = (&v) => {
63. if (v == null) { return null }
64. if (hlTypeName(v) == 'Number') { return v }
65. let a = v['$numberInt']
66. if (a == null) { a = v['$numberLong'] }
67. return a == null ? null : toNumber(a)
68. }
69. static dateOf = (&v) => {
70. if (v == null) { return null }
71. let d = v['$date']
72. return d == null ? null : numOf(d)
73. }
74. // a JSON list field that may be absent or explicitly null (most Show.cast/.seasons and nearly all Person.shows are) —
75. // iterating `null` with for-of aborts the whole run: util.hl listOr
77. // ---- tables ---------------------------------------------------------------------------
78. genresTable = new MPackDB(file = storageDir + '/genres.db', primaryKey = '@id', indexes = ['!oldId'])
79. personsTable = new MPackDB(file = storageDir + '/persons.db', primaryKey = '@id', indexes = ['!oldId'])
80. showsTable = new MPackDB(file = storageDir + '/shows.db', primaryKey = '@id', indexes = ['!oldId'])
81. seasonsTable = new MPackDB(file = storageDir + '/seasons.db', primaryKey = '@id', indexes = ['!oldId'])
82. episodesTable = new MPackDB(file = storageDir + '/episodes.db', primaryKey = '@id', indexes = ['!oldId'])
83. followsTable = new MPackDB(file = storageDir + '/follows.db', primaryKey = '@id', indexes = ['!oldId'])
84. watchesTable = new MPackDB(file = storageDir + '/watches.db', primaryKey = '@id', indexes = ['!oldId'])
86. // the new id for an old row: reuse what a previous run already gave it (a point lookup on the
87. // unique `oldId` index — proven reliable; it's a FULL table scan, find(null, null), that lost
88. // rows together with update() in testing, so this never calls that), or a fresh random id
89. static existingId = (&table, &oldId) => {
90. let found = table.find('oldId', oldId)
91. if (found != null && found.length > 0) { return found[0].id }
92. return null
93. }
94. static resolveId = (&table, &oldId) => {
95. let ex = existingId(table, oldId)
96. return ex == null ? newId() : ex
97. }
99. report = []
100. failed = 0
102. // ---- read every file once (chunked for the big three) --------------------------------
103. genreLines = linesOf(oldDir + '/Genre.jsonl')
104. personLines = linesOf(oldDir + '/Person.jsonl')
105. showLines = bigLinesOf(oldDir + '/Show.jsonl', tmpDir + '/show', 5000)
106. seasonLines = bigLinesOf(oldDir + '/Season.jsonl', tmpDir + '/season', 3000)
107. episodeLines = bigLinesOf(oldDir + '/Episode.jsonl', tmpDir + '/episode', 5000)
108. followLines = linesOf(oldDir + '/Follow.jsonl')
109. watchLines = linesOf(oldDir + '/Watch.jsonl')
111. // ---- phase 1: id maps only (every old id -> its new id, self-assigned) ---------------
112. // every reference below (Show.genres/cast/seasons, Season.show/episodes, Episode.show/season,
113. // Person.shows, Follow.oid, Watch.oid) is resolved against one of these five maps, ALL of
114. // which are complete before phase 2 builds a single record — that is what removes the need
115. // for update() (see the file header).
116. genreMap = {}
117. for (l of genreLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) genreMap[oldId] = resolveId(genresTable, oldId) }
118. personMap = {}
119. for (l of personLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) personMap[oldId] = resolveId(personsTable, oldId) }
120. showMap = {}
121. for (l of showLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) showMap[oldId] = resolveId(showsTable, oldId) }
122. seasonMap = {}
123. for (l of seasonLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) seasonMap[oldId] = resolveId(seasonsTable, oldId) }
124. episodeMap = {}
125. for (l of episodeLines) { let d = JSON.parse(l) let oldId = oidOf(d._id) episodeMap[oldId] = resolveId(episodesTable, oldId) }
127. // ---- phase 2: build and put the real records (one put() each, every reference resolved) ----
129. // ---- Genre ----------------------------------------------------------------------------
130. genreFresh = 0
131. genreKnown = 0
132. for (l of genreLines) {
133. let d = JSON.parse(l)
134. let oldId = oidOf(d._id)
135. let id = genreMap[oldId]
136. if (genresTable.fetch(id) != null) {
137. genreKnown = genreKnown + 1
138. } else {
139. let putId = genresTable.put({ id = id name = d.name tmdbId = numOf(d.tmdbId) urlSegment = d.urlSegment oldId = oldId })
140. if (putId == null) { console.log('FAIL genre ' + oldId + ': ' + genresTable.lastError()) failed = failed + 1 } else { genreFresh = genreFresh + 1 }
141. }
142. }
143. report.push('genres ' + genreFresh + ' new, ' + genreKnown + ' already there (old ' + genreLines.length + ' -> new ' + genresTable.count() + ')')
145. // ---- Person (shows resolved against showMap, already complete) -----------------------
146. personFresh = 0
147. personKnown = 0
148. personShowsMissing = 0
149. for (l of personLines) {
150. let d = JSON.parse(l)
151. let oldId = oidOf(d._id)
152. let id = personMap[oldId]
153. if (personsTable.fetch(id) != null) {
154. personKnown = personKnown + 1
155. } else {
156. let shows = []
157. for (s of listOr(d.shows)) {
158. let shid = showMap[oidOf(s._id)]
159. if (shid == null) { personShowsMissing = personShowsMissing + 1 } else { shows.push({ show = shid character = s.character }) }
160. }
161. let putId = personsTable.put({
162. id = id name = d.name tmdbId = numOf(d.tmdbId) gender = numOf(d.gender) urlSegment = d.urlSegment
163. birthDate = d.birthDate birthPlace = d.birthPlace deathDate = d.deathDate image = d.image
164. imdbId = d.imdbId shows = shows oldId = oldId
165. })
166. if (putId == null) { console.log('FAIL person ' + oldId + ': ' + personsTable.lastError()) failed = failed + 1 } else { personFresh = personFresh + 1 }
167. }
168. }
169. report.push('persons ' + personFresh + ' new, ' + personKnown + ' already there (old ' + personLines.length + ' -> new ' + personsTable.count() + '), show refs skipped ' + personShowsMissing)
171. // ---- Show (genres/cast/seasons resolved against genreMap/personMap/seasonMap) --------
172. showFresh = 0
173. showKnown = 0
174. castMissing = 0
175. showGenresMissing = 0
176. showSeasonsMissing = 0
177. for (l of showLines) {
178. let d = JSON.parse(l)
179. let oldId = oidOf(d._id)
180. let id = showMap[oldId]
181. if (showsTable.fetch(id) != null) {
182. showKnown = showKnown + 1
183. } else {
184. let genres = []
185. for (g of listOr(d.genres)) {
186. let gid = genreMap[oidOf(g._id)]
187. if (gid == null) { showGenresMissing = showGenresMissing + 1 } else { genres.push(gid) }
188. }
189. let cast = []
190. for (c of listOr(d.cast)) {
191. let pid = personMap[oidOf(c._id)]
192. if (pid == null) { castMissing = castMissing + 1 } else { cast.push({ person = pid character = c.character actor = c.actor }) }
193. }
194. let seasons = []
195. for (s of listOr(d.seasons)) {
196. let sid = seasonMap[oidOf(s)]
197. if (sid == null) { showSeasonsMissing = showSeasonsMissing + 1 } else { seasons.push(sid) }
198. }
199. let putId = showsTable.put({
200. id = id title = d.title summary = d.summary tmdbSummary = d.tmdbSummary tmdbId = numOf(d.tmdbId)
201. language = d.language type = d.type release = d.release year = numOf(d.year) image = d.image
202. urlSegment = d.urlSegment episodesCount = numOf(d.episodesCount) homepage = d.homepage
203. imdbId = d.imdbId seasonsCount = numOf(d.seasonsCount) status = d.status tagline = d.tagline
204. tvdbId = numOf(d.tvdbId) tvmzId = numOf(d.tvmzId) genres = genres cast = cast seasons = seasons
205. oldId = oldId
206. })
207. if (putId == null) { console.log('FAIL show ' + oldId + ': ' + showsTable.lastError()) failed = failed + 1 } else { showFresh = showFresh + 1 }
208. }
209. }
210. report.push('shows ' + showFresh + ' new, ' + showKnown + ' already there (old ' + showLines.length + ' -> new ' + showsTable.count() + '), cast refs skipped ' + castMissing + ', genre refs skipped ' + showGenresMissing + ', season refs skipped ' + showSeasonsMissing)
212. // ---- Season (show resolved against showMap, episodes against episodeMap) -------------
213. seasonFresh = 0
214. seasonKnown = 0
215. seasonShowMissing = 0
216. seasonEpisodesMissing = 0
217. for (l of seasonLines) {
218. let d = JSON.parse(l)
219. let oldId = oidOf(d._id)
220. let id = seasonMap[oldId]
221. if (seasonsTable.fetch(id) != null) {
222. seasonKnown = seasonKnown + 1
223. } else {
224. let show = showMap[oidOf(d.show._id)]
225. if (show == null) { seasonShowMissing = seasonShowMissing + 1 }
226. let episodes = []
227. for (e of listOr(d.episodes)) {
228. let eid = episodeMap[oidOf(e)]
229. if (eid == null) { seasonEpisodesMissing = seasonEpisodesMissing + 1 } else { episodes.push(eid) }
230. }
231. let putId = seasonsTable.put({ id = id seasonNumber = numOf(d.seasonNumber) release = d.release title = d.title summary = d.summary show = show episodes = episodes oldId = oldId })
232. if (putId == null) { console.log('FAIL season ' + oldId + ': ' + seasonsTable.lastError()) failed = failed + 1 } else { seasonFresh = seasonFresh + 1 }
233. }
234. }
235. report.push('seasons ' + seasonFresh + ' new, ' + seasonKnown + ' already there (old ' + seasonLines.length + ' -> new ' + seasonsTable.count() + '), show refs skipped ' + seasonShowMissing + ', episode refs skipped ' + seasonEpisodesMissing)
237. // ---- Episode (show/season resolved; season null where the OLD link was already null or
238. // dangling — pre-existing in the export, not introduced here) -------------------------
239. episodeFresh = 0
240. episodeKnown = 0
241. episodeShowMissing = 0
242. episodeSeasonMissing = 0
243. for (l of episodeLines) {
244. let d = JSON.parse(l)
245. let oldId = oidOf(d._id)
246. let id = episodeMap[oldId]
247. if (episodesTable.fetch(id) != null) {
248. episodeKnown = episodeKnown + 1
249. } else {
250. let show = showMap[oidOf(d.show._id)]
251. if (show == null) { episodeShowMissing = episodeShowMissing + 1 }
252. let oldSeasonId = oidOf(d.season._id)
253. let season = oldSeasonId == null ? null : seasonMap[oldSeasonId]
254. if (oldSeasonId != null && season == null) { episodeSeasonMissing = episodeSeasonMissing + 1 }
255. let putId = episodesTable.put({
256. id = id tmdbId = numOf(d.tmdbId) episodeNumber = numOf(d.episodeNumber) seasonNumber = numOf(d.seasonNumber)
257. title = d.title summary = d.summary release = d.release show = show season = season oldId = oldId
258. })
259. if (putId == null) { console.log('FAIL episode ' + oldId + ': ' + episodesTable.lastError()) failed = failed + 1 } else { episodeFresh = episodeFresh + 1 }
260. }
261. }
262. report.push('episodes ' + episodeFresh + ' new, ' + episodeKnown + ' already there (old ' + episodeLines.length + ' -> new ' + episodesTable.count() + '), show refs skipped ' + episodeShowMissing + ', season refs left null (pre-existing) ' + episodeSeasonMissing)
264. // ---- the single old user -> this app's user for ident short id az5b2 ---------------
265. creator = ensureUser('az5b2')
266. if (creator == null) { hlError('could not create/find the user for az5b2') }
267. userId = creator.id
269. // ---- Follow ---------------------------------------------------------------------
270. followFresh = 0
271. followKnown = 0
272. followMissing = 0
273. for (l of followLines) {
274. let d = JSON.parse(l)
275. let oldId = oidOf(d._id)
276. let id = resolveId(followsTable, oldId)
277. if (followsTable.fetch(id) != null) {
278. followKnown = followKnown + 1
279. } else {
280. let show = showMap[oidOf(d.oid)]
281. if (show == null) { followMissing = followMissing + 1 }
282. let putId = followsTable.put({ id = id show = show user = userId at = dateOf(d._createdAt) oldId = oldId })
283. if (putId == null) { console.log('FAIL follow ' + oldId + ': ' + followsTable.lastError()) failed = failed + 1 } else { followFresh = followFresh + 1 }
284. }
285. }
286. report.push('follows ' + followFresh + ' new, ' + followKnown + ' already there (old ' + followLines.length + ' -> new ' + followsTable.count() + '), show refs skipped ' + followMissing)
288. // ---- Watch (target null where the OLD target no longer exists — pre-existing) -----
289. watchFresh = 0
290. watchKnown = 0
291. watchMissing = 0
292. for (l of watchLines) {
293. let d = JSON.parse(l)
294. let oldId = oidOf(d._id)
295. let id = resolveId(watchesTable, oldId)
296. if (watchesTable.fetch(id) != null) {
297. watchKnown = watchKnown + 1
298. } else {
299. let kind = d.type
300. let oldTarget = oidOf(d.oid)
301. let target = kind == 'episode' ? episodeMap[oldTarget] : seasonMap[oldTarget]
302. if (target == null) { watchMissing = watchMissing + 1 }
303. let putId = watchesTable.put({ id = id targetType = kind target = target user = userId at = dateOf(d._createdAt) oldId = oldId })
304. if (putId == null) { console.log('FAIL watch ' + oldId + ': ' + watchesTable.lastError()) failed = failed + 1 } else { watchFresh = watchFresh + 1 }
305. }
306. }
307. report.push('watches ' + watchFresh + ' new, ' + watchKnown + ' already there (old ' + watchLines.length + ' -> new ' + watchesTable.count() + '), target refs left null (pre-existing) ' + watchMissing)
309. genresTable.persist()
310. personsTable.persist()
311. showsTable.persist()
312. seasonsTable.persist()
313. episodesTable.persist()
314. followsTable.persist()
315. watchesTable.persist()
317. for (line of report) { console.log('migrate: ' + line) }
318. console.log('migrate: ' + failed + ' failed (' + oldDir + ' -> ' + storageDir + ')')