tracker

6.8 KB

1. // lib/search-helpers.hl — the search's text work (search.hl), also used by the people, the merge and the franchises: a text as
2. // words (lower case, accents folded, punctuation a break), bucket keys, the query from the address (decoded only when it is
3. // valid), the ranking of hits, years, and SLUGS ("Grey's Anatomy" → "Greys-Anatomy"). Mission 028: out of search.hl.
4. import { hexDigits, textOr } from './util.hl'
6. // ---- text → words ---------------------------------------------------------------------------------
7. // lower case, apostrophes dropped ("Grey's" → "greys"), every other punctuation mark a word break, the common Latin
8. // accents folded ("Amélie" → "amelie") — the same for the titles and the query
9. static apostrophes = ["'", '’', '`', '´']
10. static breaks = ['.', ',', ':', ';', '!', '?', '(', ')', '[', ']', '{', '}', '"', '“', '”', '„', '-', '–', '—', '/', '&', '+', '*', '#', '_', '|', '…', '·', '@', '=', '~', '<', '>']
11. static folds = [['á', 'a'], ['à', 'a'], ['â', 'a'], ['ä', 'a'], ['ã', 'a'], ['å', 'a'], ['æ', 'ae'], ['ç', 'c'], ['é', 'e'], ['è', 'e'], ['ê', 'e'], ['ë', 'e'],
12. ['í', 'i'], ['ì', 'i'], ['î', 'i'], ['ï', 'i'], ['ñ', 'n'], ['ó', 'o'], ['ò', 'o'], ['ô', 'o'], ['ö', 'o'], ['õ', 'o'], ['ø', 'o'], ['œ', 'oe'],
13. ['ú', 'u'], ['ù', 'u'], ['û', 'u'], ['ü', 'u'], ['ý', 'y'], ['ÿ', 'y'], ['ß', 'ss'], ['š', 's'], ['ž', 'z'], ['č', 'c'], ['ł', 'l']]
15. static normalize = (&s) => {
16. if (s == null) { return '' }
17. let t = ('' + s).toLowerCase()
18. for (p of apostrophes) { if (t.includes(p)) { t = t.replaceAll(p, '') } }
19. for (p of breaks) { if (t.includes(p)) { t = t.replaceAll(p, ' ') } }
20. for (f of folds) { if (t.includes(f[0])) { t = t.replaceAll(f[0], f[1]) } }
21. return t
22. }
23. static wordsOf = (&s) => {
24. out = []
25. for (w of normalize(s).split(' ')) { if (w != '') { out.push(w) } }
26. return out
27. }
29. static bucketKeys = (&words) => {
30. seen = {}
31. out = []
32. for (w of words) {
33. let n = 2
34. while (n <= 3) {
35. if (w.length >= n) {
36. let k = w.slice(0, n)
37. if (seen[k] != true) { seen[k] = true out.push(k) }
38. }
39. n = n + 1
40. }
41. }
42. return out
43. }
45. static tmdbKeyOf = (&type, &tmdbId) => { return (type == 'movie' ? 'movie:' : 'series:') + tmdbId }
47. static yearWordOf = (&show) => {
48. if (show.year != null) { return '' + show.year }
49. if (show.release != null && hlTypeName(show.release) == 'String' && show.release.length >= 4) { return show.release.slice(0, 4) }
50. return ''
51. }
53. // ---- the query as it comes in the address ------------------------------------------------------------
54. // `/search/<text>`: a full page load hands the page the path segment already DECODED by the server (measured: `%2F` splits
55. // the segment → 404, so the page never puts one in the address), a client-side navigation the RAW one — decoded here.
56. // decodeURIComponent stops the request on a malformed one ('%zz', a broken UTF-8 run), so every %-run is checked first; a
57. // bad one is taken literally. (A text that itself looks like an escape, '%41', is decoded twice on a reload — accepted.)
58. static hexAt = (&s, &i) => {
59. if (i + 3 > s.length) { return -1 }
60. a = hexDigits.indexOf(s.slice(i + 1, i + 2).toLowerCase())
61. b = hexDigits.indexOf(s.slice(i + 2, i + 3).toLowerCase())
62. if (a < 0 || b < 0) { return -1 }
63. return a * 16 + b
64. }
65. // the bytes of one run of %XX are valid UTF-8 (the rules decodeURIComponent applies)
66. static utf8Ok = (&bytes) => {
67. let i = 0
68. while (i < bytes.length) {
69. let b = bytes[i]
70. let need = 0
71. let lo = 128
72. let hi = 191
73. if (b < 128) { need = 0 } else if (b >= 194 && b <= 223) { need = 1 } else if (b >= 224 && b <= 239) {
74. need = 2
75. if (b == 224) { lo = 160 }
76. if (b == 237) { hi = 159 }
77. } else if (b >= 240 && b <= 244) {
78. need = 3
79. if (b == 240) { lo = 144 }
80. if (b == 244) { hi = 143 }
81. } else { return false }
82. if (i + need > bytes.length - 1) { return false }
83. let k = 1
84. while (k <= need) {
85. let c = bytes[i + k]
86. if (k == 1 && (c < lo || c > hi)) { return false }
87. if (k > 1 && (c < 128 || c > 191)) { return false }
88. k = k + 1
89. }
90. i = i + need + 1
91. }
92. return true
93. }
94. static decodableUri = (&s) => {
95. let i = 0
96. while (i < s.length) {
97. if (s.slice(i, i + 1) == '%') {
98. let run = []
99. while (i < s.length && s.slice(i, i + 1) == '%') {
100. let v = hexAt(s, i)
101. if (v < 0) { return false }
102. run.push(v)
103. i = i + 3
104. }
105. if (!utf8Ok(run)) { return false }
106. } else {
107. i = i + 1
108. }
109. }
110. return true
111. }
112. static maxQuery = 100
113. static queryOfParam = (&raw) => {
114. if (raw == null || hlTypeName(raw) != 'String' || raw == '') { return '' }
115. t = raw.includes('%') && decodableUri(raw) ? decodeURIComponent(raw) : raw
116. return t.length > maxQuery ? t.slice(0, maxQuery) : t
117. }
119. // lower is better: the whole name (0), the name starts with the query (1), its first word does (2), anything else (3);
120. // then the shorter name
121. static scoreOf = (&e, &qw, &qtext) => {
122. if (e.text == qtext) { return 0 }
123. if (e.text.startsWith(qtext)) { return 1 }
124. if (e.words[0].startsWith(qw[0])) { return 2 }
125. return 3
126. }
127. static matchesAll = (&e, &qw) => {
128. for (q of qw) {
129. let hit = false
130. for (w of e.words) { if (!hit && w.startsWith(q)) { hit = true } }
131. if (!hit) { return false }
132. }
133. return true
134. }
135. // keeps the best `limit` of { score, len, tie, i } in order (hand-written: no list sort(), hybriel#1)
136. static insertTop = (&top, &item, &limit) => {
137. out = []
138. let inserted = false
139. for (x of top) {
140. if (!inserted && (item.score < x.score || (item.score == x.score && (item.len < x.len || (item.len == x.len && item.tie < x.tie))))) { out.push(item) inserted = true }
141. if (out.length < limit) { out.push(x) }
142. }
143. if (!inserted && out.length < limit) { out.push(item) }
144. return out
145. }
147. static yearOfDate = (&d) => { return textOr(d) != null && d.length >= 4 ? d.slice(0, 4) : '' }
149. // ---- slugs ------------------------------------------------------------------------------------------
150. static slugChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
151. // the slug of a text without the "in use" check — `fallback` when it has no Latin letter at all (people.hl personSlugOf
152. // uses it for new people, tracker#26)
153. static slugBaseOf = (&title, fallback) => {
154. t = normalizeSlugText(title)
155. let out = ''
156. let dash = false
157. let i = 0
158. while (i < t.length) {
159. let c = t.slice(i, i + 1)
160. if (slugChars.includes(c)) {
161. if (dash && out != '') { out = out + '-' }
162. out = out + c
163. dash = false
164. } else {
165. dash = true
166. }
167. i = i + 1
168. }
169. if (out == '') { out = fallback }
170. if (out.length > 80) { out = out.slice(0, 80) }
171. return out
172. }
173. // the accents folded (keeping the case) and apostrophes dropped, so "Grey's Anatomy" → "Greys-Anatomy"
174. static normalizeSlugText = (&s) => {
175. let t = '' + s
176. for (p of apostrophes) { if (t.includes(p)) { t = t.replaceAll(p, '') } }
177. for (f of folds) {
178. if (t.includes(f[0])) { t = t.replaceAll(f[0], f[1]) }
179. let up = f[0].toUpperCase()
180. if (up != f[0] && t.includes(up)) { t = t.replaceAll(up, f[1].toUpperCase()) }
181. }
182. return t
183. }