tracker

1.8 KB

1. // lib/export.hl — READING THE OLD TRACKER'S MONGO EXPORT (/media/STORAGE/projects/old-tracker/mongo-export/*.jsonl, EJSON, one
2. // document per line): the lines of a file, a Mongo id, the chunk workaround for big files. Used by the one-off tools
3. // tools/migrate.hl (tracker.worldapi.org#2) and tools/old-short-ids.hl (mission 060). Mission 028: was copied into both.
4. import { run } from 'hl:proc'
5. import { readFile, mkDir, exists } from 'hl:fs'
7. static nl = "
8. "
10. static linesOf = (&path) => {
11. raw = readFile(path)
12. if (raw == null) { return [] }
13. out = []
14. for (l of raw.split(nl)) { if (l != '') { out.push(l) } }
15. return out
16. }
17. static oidOf = (&v) => {
18. if (v == null) { return null }
19. if (hlTypeName(v) == 'String') { return v }
20. return v['$oid']
21. }
23. // hl:fs readFile SILENTLY TRUNCATES at 10 MiB (fs.zig hl_fs_read_file, max_size = 1024*1024*10,
24. // no error, no flag — reported as a Hybriel gap, tracker.worldapi.org#2): Show.jsonl (12.4 MB),
25. // Season.jsonl (10.9 MB) and Episode.jsonl (140 MB) are all over it. Worked around WITHOUT
26. // touching the vendored plugin: `split` (coreutils, via hl:proc run — no shell, no JS) cuts the
27. // export into <10 MB chunks in a scratch dir first; each chunk is read whole (well under the cap).
28. static padNum4 = (&n) => {
29. let s = '' + n
30. while (s.length < 4) { s = '0' + s }
31. return s
32. }
33. static bigLinesOf = (&path, &chunkDir, &perChunk) => {
34. mkDir(chunkDir, 448)
35. r = run(['split', '-l', '' + perChunk, '-d', '-a', '4', '--additional-suffix=.jsonl', path, chunkDir + '/c_'])
36. if (r == null || r.exit != 0) { hlError('split failed for ' + path) }
37. out = []
38. let idx = 0
39. while (true) {
40. let fp = chunkDir + '/c_' + padNum4(idx) + '.jsonl'
41. if (!exists(fp)) { break }
42. for (l of linesOf(fp)) { out.push(l) }
43. idx = idx + 1
44. }
45. if (idx == 0) { hlError('no chunks produced for ' + path) }
46. return out
47. }