tracker

7.6 KB

1. // tools/verify.hl — ONE-OFF PROOF for tracker.worldapi.org#2, run AFTER tools/migrate.hl.
2. // Never opens the live storage/mpackdb/ a second time (hl:mpackdb rewrites a file's data on
3. // open) — it works on a plain filesystem COPY the caller already made, given as
4. // TRACKER_VERIFY_COPY. Checks, independently of migrate.hl's own "skipped" counters:
5. // 1. every table's live count() equals the expected count (the old export's line count).
6. // 2. zero DANGLING references: every non-null reference field (Show.genres/cast/seasons,
7. // Season.show/episodes, Episode.show/season, Person.shows, Follow.show/user, Watch.target/
8. // user) points at an id that really exists in the target table — a pre-existing null (the
9. // old export already had no target) is not a dangling reference and is not counted here.
10. // 3. one show followed end to end: show -> its seasons -> their episodes -> a follow and a
11. // watch that reference it, printed so a human can spot-check them.
12. //
13. // Run (absolute paths; after migrate.hl, against a COPY, e.g.:
14. // rm -rf .scratch/verify-copy && cp -r storage/mpackdb .scratch/verify-copy
15. // TRACKER_VERIFY_COPY=$PWD/.scratch/verify-copy ./bin/hybriel tools/verify.hl
16. import { MPackDB } from 'hl:mpackdb'
17. import { env } from 'hl:proc'
19. dir = env('TRACKER_VERIFY_COPY')
20. if (dir == null || !dir.startsWith('/')) { hlError('set TRACKER_VERIFY_COPY (an absolute path to a COPY of storage/mpackdb — never the live one)') }
22. genresTable = new MPackDB(file = dir + '/genres.db', primaryKey = '@id', indexes = ['!oldId'])
23. personsTable = new MPackDB(file = dir + '/persons.db', primaryKey = '@id', indexes = ['!oldId'])
24. showsTable = new MPackDB(file = dir + '/shows.db', primaryKey = '@id', indexes = ['!oldId'])
25. seasonsTable = new MPackDB(file = dir + '/seasons.db', primaryKey = '@id', indexes = ['!oldId'])
26. episodesTable = new MPackDB(file = dir + '/episodes.db', primaryKey = '@id', indexes = ['!oldId'])
27. followsTable = new MPackDB(file = dir + '/follows.db', primaryKey = '@id', indexes = ['!oldId'])
28. watchesTable = new MPackDB(file = dir + '/watches.db', primaryKey = '@id', indexes = ['!oldId'])
29. usersTable = new MPackDB(file = dir + '/users.db', primaryKey = '@id', indexes = ['!identity'])
31. // expected counts: the old export's own document counts (old-tracker/README-RECONSTRUCTED.md)
32. expected = { genres = 27 persons = 15157 shows = 9453 seasons = 6430 episodes = 231584 follows = 128 watches = 11692 }
34. genres = genresTable.find(null, null)
35. persons = personsTable.find(null, null)
36. shows = showsTable.find(null, null)
37. seasons = seasonsTable.find(null, null)
38. episodes = episodesTable.find(null, null)
39. follows = followsTable.find(null, null)
40. watches = watchesTable.find(null, null)
41. users = usersTable.find(null, null)
43. countsOk = true
44. static checkCount = (&name, &got, &want) => {
45. let line = name + ': ' + got + ' (expected ' + want + ')'
46. if (got != want) { console.log('COUNT MISMATCH ' + line) return false }
47. console.log('count ok ' + line)
48. return true
49. }
50. countsOk = checkCount('genres', genres.length, expected.genres) && countsOk
51. countsOk = checkCount('persons', persons.length, expected.persons) && countsOk
52. countsOk = checkCount('shows', shows.length, expected.shows) && countsOk
53. countsOk = checkCount('seasons', seasons.length, expected.seasons) && countsOk
54. countsOk = checkCount('episodes', episodes.length, expected.episodes) && countsOk
55. countsOk = checkCount('follows', follows.length, expected.follows) && countsOk
56. countsOk = checkCount('watches', watches.length, expected.watches) && countsOk
57. console.log('users: ' + users.length + ' (expected 1, the creator)')
59. // ---- id sets (existence maps) ---------------------------------------------------------
60. static idSetOf = (&rows) => {
61. let m = {}
62. for (r of rows) { m[r.id] = true }
63. return m
64. }
65. genreIds = idSetOf(genres)
66. personIds = idSetOf(persons)
67. showIds = idSetOf(shows)
68. seasonIds = idSetOf(seasons)
69. episodeIds = idSetOf(episodes)
70. userIds = idSetOf(users)
72. dangling = 0
73. static bad = (&msg) => { console.log('DANGLING ' + msg) }
75. for (p of persons) {
76. for (s of p.shows) { if (s.show != null && showIds[s.show] == null) { bad('person ' + p.id + ' -> show ' + s.show) dangling = dangling + 1 } }
77. }
78. for (s of shows) {
79. for (g of s.genres) { if (g != null && genreIds[g] == null) { bad('show ' + s.id + ' -> genre ' + g) dangling = dangling + 1 } }
80. for (c of s.cast) { if (c.person != null && personIds[c.person] == null) { bad('show ' + s.id + ' -> person ' + c.person) dangling = dangling + 1 } }
81. for (sid of s.seasons) { if (sid != null && seasonIds[sid] == null) { bad('show ' + s.id + ' -> season ' + sid) dangling = dangling + 1 } }
82. }
83. for (s of seasons) {
84. if (s.show != null && showIds[s.show] == null) { bad('season ' + s.id + ' -> show ' + s.show) dangling = dangling + 1 }
85. for (eid of s.episodes) { if (eid != null && episodeIds[eid] == null) { bad('season ' + s.id + ' -> episode ' + eid) dangling = dangling + 1 } }
86. }
87. episodesWithNullSeason = 0
88. for (e of episodes) {
89. if (e.show != null && showIds[e.show] == null) { bad('episode ' + e.id + ' -> show ' + e.show) dangling = dangling + 1 }
90. if (e.season == null) { episodesWithNullSeason = episodesWithNullSeason + 1 } else { if (seasonIds[e.season] == null) { bad('episode ' + e.id + ' -> season ' + e.season) dangling = dangling + 1 } }
91. }
92. for (f of follows) {
93. if (f.show != null && showIds[f.show] == null) { bad('follow ' + f.id + ' -> show ' + f.show) dangling = dangling + 1 }
94. if (f.user != null && userIds[f.user] == null) { bad('follow ' + f.id + ' -> user ' + f.user) dangling = dangling + 1 }
95. }
96. watchesWithNullTarget = 0
97. for (w of watches) {
98. if (w.target == null) {
99. watchesWithNullTarget = watchesWithNullTarget + 1
100. } else {
101. let ok = w.targetType == 'episode' ? episodeIds[w.target] != null : (w.targetType == 'movie' ? showIds[w.target] != null : seasonIds[w.target] != null)
102. if (!ok) { bad('watch ' + w.id + ' -> ' + w.targetType + ' ' + w.target) dangling = dangling + 1 }
103. }
104. if (w.user != null && userIds[w.user] == null) { bad('watch ' + w.id + ' -> user ' + w.user) dangling = dangling + 1 }
105. }
107. console.log('dangling references: ' + dangling + ' (episodes with a pre-existing null season: ' + episodesWithNullSeason + ', watches with a pre-existing null target: ' + watchesWithNullTarget + ' — not dangling, the export already had no target)')
109. // ---- one show, followed end to end -----------------------------------------------------
110. static findByOldId = (&rows, &oldId) => {
111. for (r of rows) { if (r.oldId == oldId) { return r } }
112. return null
113. }
114. sample = findByOldId(shows, '60f2a92b45fc8011c774392e') // "Raised by Wolves", old Mongo _id
115. if (sample == null) {
116. console.log('END-TO-END: sample show not found by oldId')
117. } else {
118. console.log('END-TO-END show ' + sample.id + ' "' + sample.title + '" seasons=' + sample.seasons.length)
119. let totalEpisodes = 0
120. for (sid of sample.seasons) {
121. let se = null
122. for (s of seasons) { if (s.id == sid) { se = s break } }
123. if (se == null) { console.log(' MISSING season ' + sid) } else {
124. console.log(' season ' + se.seasonNumber + ' (' + se.id + ') episodes=' + se.episodes.length + ' show-back-ref-ok=' + (se.show == sample.id))
125. totalEpisodes = totalEpisodes + se.episodes.length
126. }
127. }
128. console.log(' total episodes across seasons: ' + totalEpisodes)
129. let followCount = 0
130. for (f of follows) { if (f.show == sample.id) { followCount = followCount + 1 } }
131. let watchCount = 0
132. for (w of watches) {
133. if (w.targetType == 'season') { for (sid of sample.seasons) { if (w.target == sid) { watchCount = watchCount + 1 } } }
134. }
135. console.log(' follows on this show: ' + followCount + ', season-level watches touching it: ' + watchCount)
136. }
138. console.log('VERIFY ' + (countsOk && dangling == 0 ? 'PASS' : 'FAIL'))