tracker

15.2 KB

1. // lib/search.hl — THE SEARCH (tracker.worldapi.org#14): our own database first, then "Fetch from web" (TMDB) for titles
2. // we don't have yet, and the import of the one picked. Used by components/search.hl (`/search/<text>`).
3. //
4. // dbSearch(text) our shows, movies and people — from an IN-MEMORY INDEX built once at boot (no table scan per
5. // keystroke): every title/name split into words; each word's first 2 and first 3 characters are
6. // a bucket key → the entries holding such a word. A query looks at ONE bucket (its longest word's
7. // first 3 characters, or 2 for a 2-letter word) and keeps the entries where EVERY query word is
8. // the start of one of the entry's words. A title's year is one of its words ("doctor who 2005").
9. // webSearch(text) GET <TMDB_BASE_URL>/search/multi?query=… (one request; TV + movies only, people dropped) —
10. // the titles we already have (same TMDB id and kind) are left out
11. // importTitle(kind, id) a new show record from GET /tv/<id>?append_to_response=external_ids,aggregate_credits (a movie:
12. // /movie/<id>?…,credits), then sync.hl `syncShowWith` fills it from that same answer exactly like
13. // the daily sync does (seasons, episodes, poster, external ids); added to the index. The answer goes
14. // back as `details`: the caller (components/search.hl) stores its cast + crew (credits.hl applyCredits,
15. // tracker#28 — details.hl imports this file, so it cannot be called from here)
16. //
17. // ADULT TITLES (mission 054): our database lists only titles with `adult == false` (shows.hl isPublicTitle; unknown is
18. // hidden) — `titleOk` holds that per show id, set by indexShow and by `searchTitleChanged` (the adult backfill and the daily
19. // sync call it per title). The web list asks TMDB with include_adult=false AND keeps only results with `adult` false.
20. //
21. // ONE PROCESS PER STORAGE (hybriel#21): the index lives in the app's process and is kept up to date by importTitle; the
22. // daily sync never changes a title. New rows get self-assigned 16-hex ids (hybriel#113), like sync.hl.
23. import { now } from 'hl:time'
24. import { textOr, newId } from './util.hl'
25. import { showsTable, personsTable, genresTable, isPublicTitle, kindOfTitle, titlePath, liveShowOf } from './shows.hl'
26. import { tmdbGet, tmdbImageBase, adultOf } from './tmdb.hl'
27. import { syncShowWith, persistSync } from './sync.hl'
28. import { refreshCatalogShow } from './catalog.hl'
29. import { claimShortId } from './shortids.hl'
30. import { wordsOf, bucketKeys, tmdbKeyOf, yearWordOf, scoreOf, matchesAll, insertTop, yearOfDate, slugBaseOf } from './search-helpers.hl'
32. // ---- the index --------------------------------------------------------------------------------------
33. // entries[i] = { kind ('title' | 'person'), id, words, text (the words joined by one space, without the year) }
34. static entries = []
35. static buckets = {}
36. // what we have, by TMDB id: 'series:<id>' / 'movie:<id>' → our show id (the web list leaves these out)
37. static haveTmdb = {}
38. // tracker#18 (the daily sync's TVmaze change list, dailysync.hl): TVmaze id → our show id (kept by indexShow and
39. // searchTitleChanged — the sync fills TVmaze ids)
40. static haveTvmz = {}
41. // every show slug in use (an imported title gets a free one)
42. static slugTaken = {}
43. // show id → true when the title may be listed (not adult, mission 054)
44. static titleOk = {}
45. // tracker#18 (a title renamed by the daily sync): show id → its entry's index; an entry replaced by a newer one is `stale`
46. // (skipped by dbSearch — the buckets only ever grow)
47. static titleEntry = {}
48. static staleEntry = {}
49. static indexStats = { built = false titles = 0 people = 0 keys = 0 ms = 0 }
51. // `tie` orders equally good matches: lower first (a title: series before movies, then the newer one)
52. static addEntry = (kind, id, &name, &extra, tie) => {
53. words = wordsOf(name)
54. if (words.length == 0) { return false }
55. i = entries.length
56. all = []
57. for (w of words) { all.push(w) }
58. if (extra != null && extra != '') { all.push(extra) }
59. entries.push({ kind = kind id = id words = all text = words.join(' ') tie = tie })
60. if (kind == 'title') { titleEntry[id] = i }
61. for (k of bucketKeys(all)) {
62. if (buckets[k] == null) { buckets[k] = [i] indexStats.keys = indexStats.keys + 1 } else { buckets[k].push(i) }
63. }
64. return true
65. }
68. static indexShow = (&s) => {
69. if (s.urlSegment != null) { slugTaken[s.urlSegment] = true }
70. // tracker#31: a merged tombstone keeps its slug taken, nothing else (its keeper is the title)
71. if (s.mergedInto != null) {
72. titleOk[s.id] = false
73. return null
74. }
75. if (s.tmdbId != null) { haveTmdb[tmdbKeyOf(s.type, s.tmdbId)] = s.id }
76. if (s.tvmzId != null) { haveTvmz['' + s.tvmzId] = s.id }
77. titleOk[s.id] = isPublicTitle(s)
78. y = toNumber(yearWordOf(s))
79. if (addEntry('title', s.id, s.title, yearWordOf(s), (s.type == 'movie' ? 10000 : 0) + (y != null ? 3000 - y : 2999))) { indexStats.titles = indexStats.titles + 1 }
80. return null
81. }
83. // tracker#26 (details.hl): a person added from a title's cast — findable by the search at once
84. static indexPerson = (&p) => {
85. ensureIndex()
86. if (addEntry('person', p.id, p.name, '', 0)) { indexStats.people = indexStats.people + 1 }
87. return null
88. }
90. // a title's adult flag changed (the backfill / the daily sync): listed from now on, or not
91. static searchTitleChanged = (&showId) => {
92. s = showsTable.fetch(showId)
93. if (s != null) { titleOk[s.id] = isPublicTitle(s) }
94. if (s != null && s.tvmzId != null) { haveTvmz['' + s.tvmzId] = s.id }
95. return null
96. }
98. // tracker#31 (merge.hl): a duplicate became a tombstone of `keeperId` — its entry goes stale, its TMDB id (and TVmaze id) now
99. // answer the keeper, so no import or filmography adds the title again
100. static searchTitleMerged = (&tombId, &keeperId, &type, &tmdbId) => {
101. ensureIndex()
102. old = titleEntry[tombId]
103. if (old != null) { staleEntry['' + old] = true }
104. titleOk[tombId] = false
105. if (tmdbId != null) { haveTmdb[tmdbKeyOf(type, tmdbId)] = keeperId }
106. k = showsTable.fetch(keeperId)
107. if (k != null) {
108. if (k.urlSegment != null) { slugTaken[k.urlSegment] = true }
109. if (k.tvmzId != null) { haveTvmz['' + k.tvmzId] = k.id }
110. titleOk[k.id] = isPublicTitle(k)
111. }
112. return null
113. }
115. // a title's name changed (the daily sync, tracker#18): its old entry goes stale, a new one is added — found by the new name at once
116. static searchTitleRenamed = (&showId) => {
117. if (!indexStats.built) { return null }
118. s = showsTable.fetch(showId)
119. if (s == null) { return null }
120. old = titleEntry[s.id]
121. if (old != null && entries[old].text == wordsOf(s.title).join(' ')) { return null }
122. if (old != null) { staleEntry['' + old] = true }
123. indexShow(s)
124. return null
125. }
127. // /people (tracker#32, people.hl peopleOfLetter): every person entry of the index — { id, text (the name's words, lower case) }
128. static indexedPeople = () => {
129. ensureIndex()
130. out = []
131. for (e of entries) { if (e.kind == 'person') { out.push(e) } }
132. return out
133. }
135. // built once: at boot (project.hl calls it), else by the first search
136. static ensureIndex = () => {
137. if (indexStats.built) { return true }
138. indexStats.built = true
139. t0 = now()
140. for (s of showsTable.find(null, null)) { indexShow(s) }
141. for (p of personsTable.find(null, null)) {
142. if (addEntry('person', p.id, p.name, '', 0)) { indexStats.people = indexStats.people + 1 }
143. }
144. indexStats.ms = now() - t0
145. console.log('search index: ' + indexStats.titles + ' titles, ' + indexStats.people + ' people, ' + indexStats.keys + ' keys, ' + indexStats.ms + ' ms')
146. return true
147. }
149. // tracker.worldapi.org#16 (people.hl): our show id for a TMDB title — `type` 'series' | 'movie' — or null
150. static titleIdByTmdb = (&type, &tmdbId) => {
151. ensureIndex()
152. return haveTmdb[tmdbKeyOf(type, tmdbId)]
153. }
154. // tracker#18: our show id for a TVmaze id, or null
155. static titleIdByTvmaze = (&tvmzId) => {
156. ensureIndex()
157. return haveTvmz['' + tvmzId]
158. }
160. // ---- our database ------------------------------------------------------------------------------------
161. static titleLimit = 30
162. static peopleLimit = 10
164. // { text, words, titles: [{ id }], people: [{ id }], titleCount, peopleCount, tooShort }
165. static dbSearch = (text) => {
166. ensureIndex()
167. out = { text = text titles = [] people = [] titleCount = 0 peopleCount = 0 tooShort = false ms = 0 }
168. t0 = now()
169. qw = wordsOf(text)
170. let longest = ''
171. for (w of qw) { if (w.length > longest.length) { longest = w } }
172. if (longest.length < 2) { out.tooShort = true return out }
173. key = longest.slice(0, longest.length >= 3 ? 3 : 2)
174. cand = buckets[key]
175. if (cand == null) { return out }
176. qtext = qw.join(' ')
177. let titles = []
178. let people = []
179. for (i of cand) {
180. let e = entries[i]
181. if ((e.kind != 'title' || (titleOk[e.id] == true && staleEntry['' + i] != true)) && matchesAll(e, qw)) {
182. let item = { score = scoreOf(e, qw, qtext) len = e.text.length tie = e.tie i = i }
183. if (e.kind == 'title') {
184. out.titleCount = out.titleCount + 1
185. titles = insertTop(titles, item, titleLimit)
186. } else {
187. out.peopleCount = out.peopleCount + 1
188. people = insertTop(people, item, peopleLimit)
189. }
190. }
191. }
192. for (t of titles) { out.titles.push({ id = entries[t.i].id }) }
193. for (p of people) { out.people.push({ id = entries[p.i].id }) }
194. out.ms = now() - t0
195. return out
196. }
198. // ---- the web (TMDB) -----------------------------------------------------------------------------------
200. // TMDB's search: [{ kind ('tv' | 'movie'), tmdbId, title, year, overview, posterUrl }] of what we DON'T have, or
201. // { failed }. One request (search/multi, page 1 = up to 20 results); people and titles we have are left out.
202. static webSearch = (&text) => {
203. ensureIndex()
204. qw = wordsOf(text)
205. if (text == null || text.trim().length < 2 || qw.length == 0) { return { rows = [] left = 0 failed = null } }
206. j = tmdbGet('/search/multi?query=' + encodeURIComponent(text.trim()) + '&include_adult=false&page=1')
207. if (j.failed != null) { return { rows = [] left = 0 failed = j.failed } }
208. rows = []
209. let left = 0
210. if (j.results != null) {
211. for (r of j.results) {
212. let kind = r.media_type
213. // mission 054: only what TMDB says is not adult (include_adult=false already, and the result's own flag)
214. if ((kind == 'tv' || kind == 'movie') && r.id != null && hlTypeName(r.id) == 'Number' && adultOf(r) == false) {
215. if (haveTmdb[tmdbKeyOf(kind == 'tv' ? 'series' : 'movie', r.id)] != null) {
216. left = left + 1
217. } else {
218. let title = kind == 'tv' ? textOr(r.name) : textOr(r.title)
219. let year = yearOfDate(kind == 'tv' ? r.first_air_date : r.release_date)
220. if (title != null) {
221. rows.push({ kind = kind tmdbId = r.id title = title year = year
222. overview = textOr(r.overview) != null ? r.overview : ''
223. posterUrl = textOr(r.poster_path) != null ? tmdbImageBase + '/w92' + r.poster_path : '/posters/none' })
224. }
225. }
226. }
227. }
228. }
229. return { rows = rows left = left failed = null }
230. }
232. // ---- the import ------------------------------------------------------------------------------------
233. // "Doctor Who" → "Doctor-Who" (the migrated urlSegment form: letters and digits, everything else one '-'); a slug in use
234. // gets "-2", "-3", … like the migrated ones ("Raised-by-Wolves-2"); a title with no Latin letter at all → "title-<tmdbId>"
235. static slugOf = (&title, &tmdbId) => {
236. out = slugBaseOf(title, 'title-' + tmdbId)
237. if (slugTaken[out] != true) { return out }
238. let n = 2
239. while (slugTaken[out + '-' + n] == true) { n = n + 1 }
240. return out + '-' + n
241. }
243. // our genres by lower-case name (TMDB's "Drama" → ours); TMDB-only names ("Sci-Fi & Fantasy") are left out
244. static genreIdsOf = (&tmdbGenres) => {
245. out = []
246. if (tmdbGenres == null) { return out }
247. byName = {}
248. for (g of genresTable.find(null, null)) { if (g.name != null) { byName[('' + g.name).toLowerCase()] = g.id } }
249. for (tg of tmdbGenres) {
250. let id = tg.name != null ? byName[('' + tg.name).toLowerCase()] : null
251. if (id != null && !out.includes(id)) { out.push(id) }
252. }
253. return out
254. }
256. // `kind` 'tv' | 'movie', `tmdbId` a number → { slug, path (its page, shows.hl titlePath), show, created, sync, details } or { error }. A title we already have (same
257. // TMDB id and kind) is not added again: its slug is answered.
258. static importTitle = (&kind, tmdbId) => {
259. if (kind != 'tv' && kind != 'movie') { return { error = 'unknown kind' } }
260. // `tid`, not `id`: inside the record literal below an entry named `id` would shadow it for every later entry
261. tid = toNumber('' + tmdbId)
262. if (tid == null || tid <= 0 || tid % 1 != 0) { return { error = 'bad TMDB id' } }
263. ensureIndex()
264. type = kind == 'tv' ? 'series' : 'movie'
265. have = haveTmdb[tmdbKeyOf(type, tid)]
266. if (have != null) {
267. // tracker#31: a merged tombstone answers its keeper
268. let s = liveShowOf(showsTable.fetch(have))
269. if (s != null) { return { slug = s.urlSegment path = titlePath(s) show = s.id created = false } }
270. }
271. d = tmdbGet('/' + kind + '/' + tid + '?append_to_response=external_ids,' + (kind == 'tv' ? 'aggregate_credits' : 'credits'))
272. if (d.failed != null) { return { error = 'TMDB: ' + d.failed } }
273. let title = kind == 'tv' ? textOr(d.name) : textOr(d.title)
274. if (title == null) { return { error = 'TMDB has no title for ' + kind + ' ' + tid } }
275. release = textOr(kind == 'tv' ? d.first_air_date : d.release_date)
276. year = release != null && release.length >= 4 ? toNumber(release.slice(0, 4)) : null
277. // tracker#18: `summary` is the creator's own text — TMDB's goes to tmdbSummary only
278. record = {
279. id = newId() title = title summary = null tmdbSummary = textOr(d.overview) tmdbId = tid
280. language = textOr(d.original_language) type = type release = release year = year image = null
281. urlSegment = slugOf(title, tid) episodesCount = null homepage = textOr(d.homepage) imdbId = null seasonsCount = null
282. status = textOr(d.status) tagline = textOr(d.tagline) tvdbId = null tvmzId = null genres = genreIdsOf(d.genres)
283. cast = [] seasons = [] oldId = 'tmdb-' + kind + '-' + tid imported = now() adult = adultOf(d)
284. }
285. // tracker.worldapi.org#21: a TV title's TMDB type and kind (set after the literal: an entry named `kind` would shadow the
286. // parameter for the entries after it)
287. if (kind == 'tv') {
288. record.tmdbType = textOr(d.type)
289. record.kind = kindOfTitle(record.tmdbType, record.genres)
290. }
291. // tracker#31: looked up again right before the put — the TMDB request above may have taken a while
292. again = titleIdByTmdb(type, tid)
293. if (again != null) {
294. let s = liveShowOf(showsTable.fetch(again))
295. if (s != null) { return { slug = s.urlSegment path = titlePath(s) show = s.id created = false } }
296. }
297. // mission 060: its short id (shortids.hl — unique across shows and persons)
298. record.shortId = claimShortId('show', record.id, record.oldId)
299. if (showsTable.put(record) == null) { return { error = 'could not store the title: ' + showsTable.lastError() } }
300. indexShow(record)
301. r = syncShowWith(record.id, d)
302. persistSync()
303. // the public lists (catalog.hl, #13) get it at once, like a show the daily sync touched (merge of #13 + #14)
304. refreshCatalogShow(record.id)
305. console.log('search import: ' + kind + ' ' + tid + ' "' + title + '" → ' + titlePath(record) + ' (newSeasons=' + r.newSeasons + ' newEpisodes=' + r.newEpisodes + ' posters=' + r.posters + ' ids=' + r.ids + ' requests=' + r.requests + ' errors=' + r.errors + (r.error != '' ? ' ' + r.error : '') + ')')
306. return { slug = record.urlSegment path = titlePath(record) show = record.id created = true sync = r details = d }
307. }