tracker

20.1 KB

1. // search.hl — THE SEARCH (tracker.worldapi.org#14): our own database first, then "Fetch from web" (TMDB) for titles
2. // we don't have yet, and the import of the one picked. Used by components/search.hl (`/search/<text>`).
3. //
4. // dbSearch(text) our shows, movies and people — from an IN-MEMORY INDEX built once at boot (no table scan per
5. // keystroke): every title/name split into words; each word's first 2 and first 3 characters are
6. // a bucket key → the entries holding such a word. A query looks at ONE bucket (its longest word's
7. // first 3 characters, or 2 for a 2-letter word) and keeps the entries where EVERY query word is
8. // the start of one of the entry's words. A title's year is one of its words ("doctor who 2005").
9. // webSearch(text) GET <TMDB_BASE_URL>/search/multi?query=… (one request; TV + movies only, people dropped) —
10. // the titles we already have (same TMDB id and kind) are left out
11. // importTitle(kind, id) a new show record from GET /tv/<id>?append_to_response=external_ids,aggregate_credits (a movie:
12. // /movie/<id>?…,credits), then tmdbsync.hl `syncShowWith` fills it from that same answer exactly like
13. // the daily sync does (seasons, episodes, poster, external ids); added to the index. The answer goes
14. // back as `details`: the caller (components/search.hl) stores its cast + crew (details.hl applyCredits,
15. // tracker#28 — details.hl imports this file, so it cannot be called from here)
16. //
17. // ADULT TITLES (mission 054): our database lists only titles with `adult == false` (shows.hl isPublicTitle; unknown is
18. // hidden) — `titleOk` holds that per show id, set by indexShow and by `searchTitleChanged` (the adult backfill and the daily
19. // sync call it per title). The web list asks TMDB with include_adult=false AND keeps only results with `adult` false.
20. //
21. // ONE PROCESS PER STORAGE (hybriel#21): the index lives in the app's process and is kept up to date by importTitle; the
22. // daily sync never changes a title. New rows get self-assigned 16-hex ids (hybriel#113), like tmdbsync.hl.
23. import { now } from 'hl:time'
24. import { randomBytes } from 'hl:crypto'
25. import { showsTable, personsTable, genresTable, isPublicTitle, kindOfTitle, titlePath } from './shows.hl'
26. import { tmdbGet, tmdbImageBase, syncShowWith, persistSync, textOr, adultOf } from './tmdbsync.hl'
27. import { refreshCatalogShow } from './catalog.hl'
28. import { claimShortId } from './shortids.hl'
30. // ---- text → words ---------------------------------------------------------------------------------
31. // lower case, apostrophes dropped ("Grey's" → "greys"), every other punctuation mark a word break, the common Latin
32. // accents folded ("Amélie" → "amelie") — the same for the titles and the query
33. static apostrophes = ["'", '’', '`', '´']
34. static breaks = ['.', ',', ':', ';', '!', '?', '(', ')', '[', ']', '{', '}', '"', '“', '”', '„', '-', '–', '—', '/', '&', '+', '*', '#', '_', '|', '…', '·', '@', '=', '~', '<', '>']
35. static folds = [['á', 'a'], ['à', 'a'], ['â', 'a'], ['ä', 'a'], ['ã', 'a'], ['å', 'a'], ['æ', 'ae'], ['ç', 'c'], ['é', 'e'], ['è', 'e'], ['ê', 'e'], ['ë', 'e'],
36. ['í', 'i'], ['ì', 'i'], ['î', 'i'], ['ï', 'i'], ['ñ', 'n'], ['ó', 'o'], ['ò', 'o'], ['ô', 'o'], ['ö', 'o'], ['õ', 'o'], ['ø', 'o'], ['œ', 'oe'],
37. ['ú', 'u'], ['ù', 'u'], ['û', 'u'], ['ü', 'u'], ['ý', 'y'], ['ÿ', 'y'], ['ß', 'ss'], ['š', 's'], ['ž', 'z'], ['č', 'c'], ['ł', 'l']]
39. static normalize = (&s) => {
40. if (s == null) { return '' }
41. let t = ('' + s).toLowerCase()
42. for (p of apostrophes) { if (t.includes(p)) { t = t.replaceAll(p, '') } }
43. for (p of breaks) { if (t.includes(p)) { t = t.replaceAll(p, ' ') } }
44. for (f of folds) { if (t.includes(f[0])) { t = t.replaceAll(f[0], f[1]) } }
45. return t
46. }
47. static wordsOf = (&s) => {
48. let out = []
49. for (w of normalize(s).split(' ')) { if (w != '') { out.push(w) } }
50. return out
51. }
53. // ---- the index --------------------------------------------------------------------------------------
54. // entries[i] = { kind ('title' | 'person'), id, words, text (the words joined by one space, without the year) }
55. static entries = []
56. static buckets = {}
57. // what we have, by TMDB id: 'series:<id>' / 'movie:<id>' → our show id (the web list leaves these out)
58. static haveTmdb = {}
59. // tracker#18 (the daily sync's TVmaze change list, deltasync.hl): TVmaze id → our show id (kept by indexShow and
60. // searchTitleChanged — the sync fills TVmaze ids)
61. static haveTvmz = {}
62. // every show slug in use (an imported title gets a free one)
63. static slugTaken = {}
64. // show id → true when the title may be listed (not adult, mission 054)
65. static titleOk = {}
66. // tracker#18 (a title renamed by the daily sync): show id → its entry's index; an entry replaced by a newer one is `stale`
67. // (skipped by dbSearch — the buckets only ever grow)
68. static titleEntry = {}
69. static staleEntry = {}
70. static indexStats = { built = false titles = 0 people = 0 keys = 0 ms = 0 }
72. static bucketKeys = (&words) => {
73. let seen = {}
74. let out = []
75. for (w of words) {
76. let n = 2
77. while (n <= 3) {
78. if (w.length >= n) {
79. let k = w.slice(0, n)
80. if (seen[k] != true) { seen[k] = true out.push(k) }
81. }
82. n = n + 1
83. }
84. }
85. return out
86. }
88. // `tie` orders equally good matches: lower first (a title: series before movies, then the newer one)
89. static addEntry = (kind, id, &name, &extra, tie) => {
90. let words = wordsOf(name)
91. if (words.length == 0) { return false }
92. let i = entries.length
93. let all = []
94. for (w of words) { all.push(w) }
95. if (extra != null && extra != '') { all.push(extra) }
96. entries.push({ kind = kind id = id words = all text = words.join(' ') tie = tie })
97. if (kind == 'title') { titleEntry[id] = i }
98. for (k of bucketKeys(all)) {
99. if (buckets[k] == null) { buckets[k] = [i] indexStats.keys = indexStats.keys + 1 } else { buckets[k].push(i) }
100. }
101. return true
102. }
104. static tmdbKeyOf = (&type, &tmdbId) => { return (type == 'movie' ? 'movie:' : 'series:') + tmdbId }
106. static yearWordOf = (&show) => {
107. if (show.year != null) { return '' + show.year }
108. if (show.release != null && hlTypeName(show.release) == 'String' && show.release.length >= 4) { return show.release.slice(0, 4) }
109. return ''
110. }
112. static indexShow = (&s) => {
113. if (s.urlSegment != null) { slugTaken[s.urlSegment] = true }
114. if (s.tmdbId != null) { haveTmdb[tmdbKeyOf(s.type, s.tmdbId)] = s.id }
115. if (s.tvmzId != null) { haveTvmz['' + s.tvmzId] = s.id }
116. titleOk[s.id] = isPublicTitle(s)
117. let y = toNumber(yearWordOf(s))
118. if (addEntry('title', s.id, s.title, yearWordOf(s), (s.type == 'movie' ? 10000 : 0) + (y != null ? 3000 - y : 2999))) { indexStats.titles = indexStats.titles + 1 }
119. return null
120. }
122. // tracker#26 (details.hl): a person added from a title's cast — findable by the search at once
123. static indexPerson = (&p) => {
124. ensureIndex()
125. if (addEntry('person', p.id, p.name, '', 0)) { indexStats.people = indexStats.people + 1 }
126. return null
127. }
129. // a title's adult flag changed (the backfill / the daily sync): listed from now on, or not
130. static searchTitleChanged = (&showId) => {
131. let s = showsTable.fetch(showId)
132. if (s != null) { titleOk[s.id] = isPublicTitle(s) }
133. if (s != null && s.tvmzId != null) { haveTvmz['' + s.tvmzId] = s.id }
134. return null
135. }
137. // a title's name changed (the daily sync, tracker#18): its old entry goes stale, a new one is added — found by the new name at once
138. static searchTitleRenamed = (&showId) => {
139. if (!indexStats.built) { return null }
140. let s = showsTable.fetch(showId)
141. if (s == null) { return null }
142. let old = titleEntry[s.id]
143. if (old != null && entries[old].text == wordsOf(s.title).join(' ')) { return null }
144. if (old != null) { staleEntry['' + old] = true }
145. indexShow(s)
146. return null
147. }
149. // built once: at boot (project.hl calls it), else by the first search
150. static ensureIndex = () => {
151. if (indexStats.built) { return true }
152. indexStats.built = true
153. let t0 = now()
154. for (s of showsTable.find(null, null)) { indexShow(s) }
155. for (p of personsTable.find(null, null)) {
156. if (addEntry('person', p.id, p.name, '', 0)) { indexStats.people = indexStats.people + 1 }
157. }
158. indexStats.ms = now() - t0
159. console.log('search index: ' + indexStats.titles + ' titles, ' + indexStats.people + ' people, ' + indexStats.keys + ' keys, ' + indexStats.ms + ' ms')
160. return true
161. }
163. // tracker.worldapi.org#16 (people.hl): our show id for a TMDB title — `type` 'series' | 'movie' — or null
164. static titleIdByTmdb = (&type, &tmdbId) => {
165. ensureIndex()
166. return haveTmdb[tmdbKeyOf(type, tmdbId)]
167. }
168. // tracker#18: our show id for a TVmaze id, or null
169. static titleIdByTvmaze = (&tvmzId) => {
170. ensureIndex()
171. return haveTvmz['' + tvmzId]
172. }
174. // ---- the query as it comes in the address ------------------------------------------------------------
175. // `/search/<text>`: a full page load hands the page the path segment already DECODED by the server (measured: `%2F` splits
176. // the segment → 404, so the page never puts one in the address), a client-side navigation the RAW one — decoded here.
177. // decodeURIComponent stops the request on a malformed one ('%zz', a broken UTF-8 run), so every %-run is checked first; a
178. // bad one is taken literally. (A text that itself looks like an escape, '%41', is decoded twice on a reload — accepted.)
179. static hexDigits = '0123456789abcdef'
180. static hexAt = (&s, &i) => {
181. if (i + 3 > s.length) { return -1 }
182. let a = hexDigits.indexOf(s.slice(i + 1, i + 2).toLowerCase())
183. let b = hexDigits.indexOf(s.slice(i + 2, i + 3).toLowerCase())
184. if (a < 0 || b < 0) { return -1 }
185. return a * 16 + b
186. }
187. // the bytes of one run of %XX are valid UTF-8 (the rules decodeURIComponent applies)
188. static utf8Ok = (&bytes) => {
189. let i = 0
190. while (i < bytes.length) {
191. let b = bytes[i]
192. let need = 0
193. let lo = 128
194. let hi = 191
195. if (b < 128) { need = 0 } else if (b >= 194 && b <= 223) { need = 1 } else if (b >= 224 && b <= 239) {
196. need = 2
197. if (b == 224) { lo = 160 }
198. if (b == 237) { hi = 159 }
199. } else if (b >= 240 && b <= 244) {
200. need = 3
201. if (b == 240) { lo = 144 }
202. if (b == 244) { hi = 143 }
203. } else { return false }
204. if (i + need > bytes.length - 1) { return false }
205. let k = 1
206. while (k <= need) {
207. let c = bytes[i + k]
208. if (k == 1 && (c < lo || c > hi)) { return false }
209. if (k > 1 && (c < 128 || c > 191)) { return false }
210. k = k + 1
211. }
212. i = i + need + 1
213. }
214. return true
215. }
216. static decodableUri = (&s) => {
217. let i = 0
218. while (i < s.length) {
219. if (s.slice(i, i + 1) == '%') {
220. let run = []
221. while (i < s.length && s.slice(i, i + 1) == '%') {
222. let v = hexAt(s, i)
223. if (v < 0) { return false }
224. run.push(v)
225. i = i + 3
226. }
227. if (!utf8Ok(run)) { return false }
228. } else {
229. i = i + 1
230. }
231. }
232. return true
233. }
234. static maxQuery = 100
235. static queryOfParam = (&raw) => {
236. if (raw == null || hlTypeName(raw) != 'String' || raw == '') { return '' }
237. let t = raw.includes('%') && decodableUri(raw) ? decodeURIComponent(raw) : raw
238. return t.length > maxQuery ? t.slice(0, maxQuery) : t
239. }
241. // ---- our database ------------------------------------------------------------------------------------
242. static titleLimit = 30
243. static peopleLimit = 10
245. // lower is better: the whole name (0), the name starts with the query (1), its first word does (2), anything else (3);
246. // then the shorter name
247. static scoreOf = (&e, &qw, &qtext) => {
248. if (e.text == qtext) { return 0 }
249. if (e.text.startsWith(qtext)) { return 1 }
250. if (e.words[0].startsWith(qw[0])) { return 2 }
251. return 3
252. }
253. static matchesAll = (&e, &qw) => {
254. for (q of qw) {
255. let hit = false
256. for (w of e.words) { if (!hit && w.startsWith(q)) { hit = true } }
257. if (!hit) { return false }
258. }
259. return true
260. }
261. // keeps the best `limit` of { score, len, tie, i } in order (hand-written: no list sort(), hybriel#1)
262. static insertTop = (&top, &item, &limit) => {
263. let out = []
264. let inserted = false
265. for (x of top) {
266. if (!inserted && (item.score < x.score || (item.score == x.score && (item.len < x.len || (item.len == x.len && item.tie < x.tie))))) { out.push(item) inserted = true }
267. if (out.length < limit) { out.push(x) }
268. }
269. if (!inserted && out.length < limit) { out.push(item) }
270. return out
271. }
273. // { text, words, titles: [{ id }], people: [{ id }], titleCount, peopleCount, tooShort }
274. static dbSearch = (text) => {
275. ensureIndex()
276. let out = { text = text titles = [] people = [] titleCount = 0 peopleCount = 0 tooShort = false ms = 0 }
277. let t0 = now()
278. let qw = wordsOf(text)
279. let longest = ''
280. for (w of qw) { if (w.length > longest.length) { longest = w } }
281. if (longest.length < 2) { out.tooShort = true return out }
282. let key = longest.slice(0, longest.length >= 3 ? 3 : 2)
283. let cand = buckets[key]
284. if (cand == null) { return out }
285. let qtext = qw.join(' ')
286. let titles = []
287. let people = []
288. for (i of cand) {
289. let e = entries[i]
290. if ((e.kind != 'title' || (titleOk[e.id] == true && staleEntry['' + i] != true)) && matchesAll(e, qw)) {
291. let item = { score = scoreOf(e, qw, qtext) len = e.text.length tie = e.tie i = i }
292. if (e.kind == 'title') {
293. out.titleCount = out.titleCount + 1
294. titles = insertTop(titles, item, titleLimit)
295. } else {
296. out.peopleCount = out.peopleCount + 1
297. people = insertTop(people, item, peopleLimit)
298. }
299. }
300. }
301. for (t of titles) { out.titles.push({ id = entries[t.i].id }) }
302. for (p of people) { out.people.push({ id = entries[p.i].id }) }
303. out.ms = now() - t0
304. return out
305. }
307. // ---- the web (TMDB) -----------------------------------------------------------------------------------
308. static yearOfDate = (&d) => { return textOr(d) != null && d.length >= 4 ? d.slice(0, 4) : '' }
310. // TMDB's search: [{ kind ('tv' | 'movie'), tmdbId, title, year, overview, posterUrl }] of what we DON'T have, or
311. // { failed }. One request (search/multi, page 1 = up to 20 results); people and titles we have are left out.
312. static webSearch = (&text) => {
313. ensureIndex()
314. let qw = wordsOf(text)
315. if (text == null || text.trim().length < 2 || qw.length == 0) { return { rows = [] left = 0 failed = null } }
316. let j = tmdbGet('/search/multi?query=' + encodeURIComponent(text.trim()) + '&include_adult=false&page=1')
317. if (j.failed != null) { return { rows = [] left = 0 failed = j.failed } }
318. let rows = []
319. let left = 0
320. if (j.results != null) {
321. for (r of j.results) {
322. let kind = r.media_type
323. // mission 054: only what TMDB says is not adult (include_adult=false already, and the result's own flag)
324. if ((kind == 'tv' || kind == 'movie') && r.id != null && hlTypeName(r.id) == 'Number' && adultOf(r) == false) {
325. if (haveTmdb[tmdbKeyOf(kind == 'tv' ? 'series' : 'movie', r.id)] != null) {
326. left = left + 1
327. } else {
328. let title = kind == 'tv' ? textOr(r.name) : textOr(r.title)
329. let year = yearOfDate(kind == 'tv' ? r.first_air_date : r.release_date)
330. if (title != null) {
331. rows.push({ kind = kind tmdbId = r.id title = title year = year
332. overview = textOr(r.overview) != null ? r.overview : ''
333. posterUrl = textOr(r.poster_path) != null ? tmdbImageBase + '/w92' + r.poster_path : '/posters/none' })
334. }
335. }
336. }
337. }
338. }
339. return { rows = rows left = left failed = null }
340. }
342. // ---- the import ------------------------------------------------------------------------------------
343. static slugChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
344. // "Doctor Who" → "Doctor-Who" (the migrated urlSegment form: letters and digits, everything else one '-'); a slug in use
345. // gets "-2", "-3", … like the migrated ones ("Raised-by-Wolves-2"); a title with no Latin letter at all → "title-<tmdbId>"
346. static slugOf = (&title, &tmdbId) => {
347. let out = slugBaseOf(title, 'title-' + tmdbId)
348. if (slugTaken[out] != true) { return out }
349. let n = 2
350. while (slugTaken[out + '-' + n] == true) { n = n + 1 }
351. return out + '-' + n
352. }
353. // the slug of a text without the "in use" check — `fallback` when it has no Latin letter at all (people.hl personSlugOf
354. // uses it for new people, tracker#26)
355. static slugBaseOf = (&title, fallback) => {
356. let t = normalizeSlugText(title)
357. let out = ''
358. let dash = false
359. let i = 0
360. while (i < t.length) {
361. let c = t.slice(i, i + 1)
362. if (slugChars.includes(c)) {
363. if (dash && out != '') { out = out + '-' }
364. out = out + c
365. dash = false
366. } else {
367. dash = true
368. }
369. i = i + 1
370. }
371. if (out == '') { out = fallback }
372. if (out.length > 80) { out = out.slice(0, 80) }
373. return out
374. }
375. // the accents folded (keeping the case) and apostrophes dropped, so "Grey's Anatomy" → "Greys-Anatomy"
376. static normalizeSlugText = (&s) => {
377. let t = '' + s
378. for (p of apostrophes) { if (t.includes(p)) { t = t.replaceAll(p, '') } }
379. for (f of folds) {
380. if (t.includes(f[0])) { t = t.replaceAll(f[0], f[1]) }
381. let up = f[0].toUpperCase()
382. if (up != f[0] && t.includes(up)) { t = t.replaceAll(up, f[1].toUpperCase()) }
383. }
384. return t
385. }
387. // our genres by lower-case name (TMDB's "Drama" → ours); TMDB-only names ("Sci-Fi & Fantasy") are left out
388. static genreIdsOf = (&tmdbGenres) => {
389. let out = []
390. if (tmdbGenres == null) { return out }
391. let byName = {}
392. for (g of genresTable.find(null, null)) { if (g.name != null) { byName[('' + g.name).toLowerCase()] = g.id } }
393. for (tg of tmdbGenres) {
394. let id = tg.name != null ? byName[('' + tg.name).toLowerCase()] : null
395. if (id != null && !out.includes(id)) { out.push(id) }
396. }
397. return out
398. }
400. // `kind` 'tv' | 'movie', `tmdbId` a number → { slug, path (its page, shows.hl titlePath), show, created, sync, details } or { error }. A title we already have (same
401. // TMDB id and kind) is not added again: its slug is answered.
402. static importTitle = (&kind, tmdbId) => {
403. if (kind != 'tv' && kind != 'movie') { return { error = 'unknown kind' } }
404. // `tid`, not `id`: inside the record literal below an entry named `id` would shadow it for every later entry
405. let tid = toNumber('' + tmdbId)
406. if (tid == null || tid <= 0 || tid % 1 != 0) { return { error = 'bad TMDB id' } }
407. ensureIndex()
408. let type = kind == 'tv' ? 'series' : 'movie'
409. let have = haveTmdb[tmdbKeyOf(type, tid)]
410. if (have != null) {
411. let s = showsTable.fetch(have)
412. if (s != null) { return { slug = s.urlSegment path = titlePath(s) show = s.id created = false } }
413. }
414. let d = tmdbGet('/' + kind + '/' + tid + '?append_to_response=external_ids,' + (kind == 'tv' ? 'aggregate_credits' : 'credits'))
415. if (d.failed != null) { return { error = 'TMDB: ' + d.failed } }
416. let title = kind == 'tv' ? textOr(d.name) : textOr(d.title)
417. if (title == null) { return { error = 'TMDB has no title for ' + kind + ' ' + tid } }
418. let release = textOr(kind == 'tv' ? d.first_air_date : d.release_date)
419. let year = release != null && release.length >= 4 ? toNumber(release.slice(0, 4)) : null
420. // tracker#18: `summary` is the creator's own text — TMDB's goes to tmdbSummary only
421. let record = {
422. id = randomBytes(8, 'hex') title = title summary = null tmdbSummary = textOr(d.overview) tmdbId = tid
423. language = textOr(d.original_language) type = type release = release year = year image = null
424. urlSegment = slugOf(title, tid) episodesCount = null homepage = textOr(d.homepage) imdbId = null seasonsCount = null
425. status = textOr(d.status) tagline = textOr(d.tagline) tvdbId = null tvmzId = null genres = genreIdsOf(d.genres)
426. cast = [] seasons = [] oldId = 'tmdb-' + kind + '-' + tid imported = now() adult = adultOf(d)
427. }
428. // tracker.worldapi.org#21: a TV title's TMDB type and kind (set after the literal: an entry named `kind` would shadow the
429. // parameter for the entries after it)
430. if (kind == 'tv') {
431. record.tmdbType = textOr(d.type)
432. record.kind = kindOfTitle(record.tmdbType, record.genres)
433. }
434. // mission 060: its short id (shortids.hl — unique across shows and persons)
435. record.shortId = claimShortId('show', record.id, record.oldId)
436. if (showsTable.put(record) == null) { return { error = 'could not store the title: ' + showsTable.lastError() } }
437. indexShow(record)
438. let r = syncShowWith(record.id, d)
439. persistSync()
440. // the public lists (catalog.hl, #13) get it at once, like a show the daily sync touched (merge of #13 + #14)
441. refreshCatalogShow(record.id)
442. console.log('search import: ' + kind + ' ' + tid + ' "' + title + '" → ' + titlePath(record) + ' (newSeasons=' + r.newSeasons + ' newEpisodes=' + r.newEpisodes + ' posters=' + r.posters + ' ids=' + r.ids + ' requests=' + r.requests + ' errors=' + r.errors + (r.error != '' ? ' ' + r.error : '') + ')')
443. return { slug = record.urlSegment path = titlePath(record) show = record.id created = true sync = r details = d }
444. }