tracker

2.8 KB

1. // tools/count-short-ids.hl — mission 060 (tracker.worldapi.org#27): the short ids in a storage, counted. ONLY on a COPY or with
2. // the app stopped (opening a table rewrites its files, hybriel#21):
3. // TRACKER_STORAGE=$PWD/.scratch/realdata/storage/mpackdb TRACKER_OLD_SHORT_IDS=$PWD/data/old-short-ids.json ./bin/hybriel tools/count-short-ids.hl
4. // → per kind: rows, with a short id, without, malformed; across both: duplicates; the old ids: kept (the show with that oldId
5. // has it), not yet (it has none), lost (it has another one), held by another record; new ids that are old ones.
6. import { MPackDB } from 'hl:mpackdb'
7. import { env } from 'hl:proc'
8. import { readFile } from 'hl:fs'
10. dir = env('TRACKER_STORAGE')
11. oldFile = env('TRACKER_OLD_SHORT_IDS')
12. if (dir == null || !dir.startsWith('/') || oldFile == null || !oldFile.startsWith('/')) { hlError('set TRACKER_STORAGE and TRACKER_OLD_SHORT_IDS, both ABSOLUTE') }
13. chars = 'abcdefghijklmnopqrstuvwxyz0123456789'
14. isShortId = (s) => {
15. if (s == null || hlTypeName(s) != 'String' || s.length != 5) { return false }
16. let i = 0
17. while (i < 5) {
18. if (!chars.includes(s[i])) { return false }
19. i = i + 1
20. }
21. return true
22. }
23. old = JSON.parse(readFile(oldFile))
24. oldSet = {}
25. oldCount = 0
26. for (k of old.keys()) { oldSet[old[k]] = k oldCount = oldCount + 1 }
28. seen = {}
29. dups = 0
30. oldKept = 0
31. oldLost = 0
32. oldPending = 0
33. oldHeldByOther = 0
34. showsWithOldIdRow = 0
35. for (kind of ['show', 'person']) {
36. let db = new MPackDB(file = dir + '/' + (kind == 'show' ? 'shows' : 'persons') + '.db', primaryKey = '@id', indexes = ['!oldId'])
37. let rows = 0
38. let have = 0
39. let none = 0
40. let bad = 0
41. let sample = ''
42. for (r of db.find(null, null)) {
43. rows = rows + 1
44. let sid = r.shortId
45. if (sid == null) { none = none + 1 } else if (!isShortId(sid)) { bad = bad + 1 } else {
46. have = have + 1
47. if (seen[sid] != null) { dups = dups + 1 print('DUPLICATE ' + sid + ': ' + seen[sid] + ' and ' + kind + ' ' + r.id) } else { seen[sid] = kind + ' ' + r.id }
48. if (sample == '' && (kind == 'person' || old[r.oldId] == null)) { sample = sid + ' → ' + r.urlSegment }
49. }
50. if (kind == 'show' && r.oldId != null && old[r.oldId] != null) {
51. showsWithOldIdRow = showsWithOldIdRow + 1
52. if (sid == old[r.oldId]) { oldKept = oldKept + 1 } else if (sid == null) { oldPending = oldPending + 1 } else { oldLost = oldLost + 1 print('OLD LOST ' + r.oldId + ': wants ' + old[r.oldId] + ', has ' + sid) }
53. } else if (sid != null && oldSet[sid] != null) { oldHeldByOther = oldHeldByOther + 1 print('OLD ID ON ANOTHER RECORD ' + sid + ': ' + kind + ' ' + r.id) }
54. }
55. print(kind + 's rows=' + rows + ' withShortId=' + have + ' without=' + none + ' malformed=' + bad + ' sample ' + sample)
56. }
57. print('old ids in the file=' + oldCount + ' shows with such an oldId=' + showsWithOldIdRow + ' kept=' + oldKept + ' notYet=' + oldPending + ' lost=' + oldLost + ' oldIdOnAnotherRecord=' + oldHeldByOther)
58. print('duplicates across shows+persons=' + dups)