tracker

2.5 KB

1. // tools/old-short-ids.hl — mission 060 (tracker.worldapi.org#27), one-off: the old tracker's short ids → data/old-short-ids.json.
2. // The old export's Show.jsonl carries `id` (5 characters, base62 counter '100jh'…) on 702 of 9,453 shows; the file maps the
3. // old Mongo `_id` (= our shows' `oldId`, tools/migrate.hl) → that id. shortids.hl reads it at start and gives those shows
4. // their old id back. Reads only the export (never storage/); the export is on Loreana only, the JSON file goes with the code.
5. //
6. // TRACKER_OLD_DATA=/media/STORAGE/projects/old-tracker/mongo-export TRACKER_MIGRATE_TMP=$PWD/.scratch/old-short-ids-tmp \
7. // TRACKER_OLD_SHORT_IDS=$PWD/data/old-short-ids.json ./bin/hybriel tools/old-short-ids.hl
8. import { env, run } from 'hl:proc'
9. import { readFile, writeFile, mkDir, exists } from 'hl:fs'
11. static nl = "
12. "
14. oldDir = env('TRACKER_OLD_DATA')
15. tmpDir = env('TRACKER_MIGRATE_TMP')
16. outFile = env('TRACKER_OLD_SHORT_IDS')
17. if (oldDir == null || !oldDir.startsWith('/') || tmpDir == null || !tmpDir.startsWith('/') || outFile == null || !outFile.startsWith('/')) {
18. hlError('set TRACKER_OLD_DATA (the mongo-export directory), TRACKER_MIGRATE_TMP (a scratch directory) and TRACKER_OLD_SHORT_IDS (the JSON file to write), all ABSOLUTE')
19. }
21. static linesOf = (path) => {
22. let raw = readFile(path)
23. if (raw == null) { return [] }
24. let out = []
25. for (l of raw.split(nl)) { if (l != '') { out.push(l) } }
26. return out
27. }
28. static oidOf = (v) => {
29. if (v == null) { return null }
30. if (hlTypeName(v) == 'String') { return v }
31. return v['$oid']
32. }
33. // hl:fs readFile stops at 10 MiB (tools/migrate.hl): Show.jsonl (12.4 MB) is cut into chunks first, like migrate.hl does
34. static padNum4 = (n) => {
35. let s = '' + n
36. while (s.length < 4) { s = '0' + s }
37. return s
38. }
39. static bigLinesOf = (path, chunkDir, perChunk) => {
40. mkDir(chunkDir, 448)
41. let r = run(['split', '-l', '' + perChunk, '-d', '-a', '4', '--additional-suffix=.jsonl', path, chunkDir + '/c_'])
42. if (r == null || r.exit != 0) { hlError('split failed for ' + path) }
43. let out = []
44. let idx = 0
45. while (true) {
46. let fp = chunkDir + '/c_' + padNum4(idx) + '.jsonl'
47. if (!exists(fp)) { break }
48. for (l of linesOf(fp)) { out.push(l) }
49. idx = idx + 1
50. }
51. return out
52. }
54. shows = 0
55. found = {}
56. count = 0
57. for (l of bigLinesOf(oldDir + '/Show.jsonl', tmpDir + '/show', 5000)) {
58. let d = JSON.parse(l)
59. shows = shows + 1
60. if (d.id != null && hlTypeName(d.id) == 'String' && d.id != '') {
61. found[oidOf(d._id)] = d.id
62. count = count + 1
63. }
64. }
65. writeFile(outFile, JSON.stringify(found), 420)
66. print('shows=' + shows + ' withShortId=' + count + ' → ' + outFile)