tracker

19.2 KB

1. // project.hl — tracker.worldapi.org: THE MAP (mission 028, docs: README "Files"). Config, the routes and how the parts are wired;
2. // what each feature does is in the file named here, not in this one.
3. //
4. // login (ident only, the header's selector + button) lib/users.hl, components/main.hl, login.js, /login/* below
5. // the shell: header, main menu, user menu, sidebar components/main.hl (theme/shell/*, vendored from layouts.worldapi.org)
6. // /genres · /people (the main menu) · /settings lib/catalog.hl genreList · lib/people.hl peoplePage · components/settings.hl
7. // titles: records, slugs, kinds, list caches lib/shows.hl a title's page: components/show.hl
8. // public lists /movies /series /shows /genres, home lib/catalog.hl
9. // components/{movies,allseries,allshows,genre,home,tilelist}.hl
10. // follows · watches · /my/* lib/follows.hl · lib/watches.hl
11. // components/{myshows,mymovies,unwatched,schedule}.hl
12. // search, web search, import lib/search.hl (+ search-helpers.hl) components/search.hl
13. // people, a person's page and filmography fill lib/people.hl components/person.hl
14. // cast + crew, the credits job lib/credits.hl (+ credits-helpers.hl)
15. // a title completed from TMDB (details repair) lib/details.hl
16. // TMDB / TVmaze requests, posters lib/tmdb.hl
17. // the sync of one title · which titles a day lib/sync.hl (+ sync-helpers.hl) · lib/dailysync.hl
18. // adult flag + kind backfills lib/backfill.hl
19. // short ids (/<shortId>) lib/shortids.hl
20. // franchises + timelines · TMDB collection seed lib/franchises.hl · lib/collections.hl
21. // the curated franchises (data/franchises.json, #40) lib/franchiseseed.hl
22. // components/franchise*.hl, timelineedit.hl
23. // duplicate titles merged lib/merge.hl
24. // the background jobs and their clocks lib/jobs.hl (one instance, made below after the server)
25. // poster + photo files lib/images.hl
26. // shared helpers (env, paths, dates, ids, lists) lib/util.hl
27. // styles · design tokens · the WorldAPI theme components/styles.hl · shared/tokens.hl · theme/ (never edited here)
28. //
29. // Config (env; no committed secret — the first deploy registers this app in ident and sets these, done by the architect, as
30. // with ident's own .env for SMTP): TRACKER_PORT (45008), TRACKER_URL (this app's own public address, for the return URL),
31. // IDENT_URL (https://ident.worldapi.org), TRACKER_KEY (pk_…), TRACKER_SECRET (sk_…), TRACKER_STORAGE, TRACKER_SESSIONS,
32. // TRACKER_WATCH, TMDB_READ_TOKEN; the jobs' switches and limits: lib/jobs.hl (README "Config").
33. import WebFramework from 'hl:web'
34. import { Response } from 'hl:http1'
35. import { env } from 'hl:proc'
36. import { now } from 'hl:time'
38. import Styles from './components/styles.hl'
39. import Home from './components/home.hl'
40. import LoginFailed from './components/loginfailed.hl'
41. import Show from './components/show.hl'
42. import Unwatched from './components/unwatched.hl'
43. import Schedule from './components/schedule.hl'
44. import MyShows from './components/myshows.hl'
45. import MyMovies from './components/mymovies.hl'
46. import Movies from './components/movies.hl'
47. import AllShows from './components/allshows.hl'
48. import AllSeries from './components/allseries.hl'
49. import Search from './components/search.hl'
50. import Person from './components/person.hl'
51. import Genre from './components/genre.hl'
52. import Genres from './components/genres.hl'
53. import People from './components/people.hl'
54. import Settings from './components/settings.hl'
55. import Franchises from './components/franchises.hl'
56. import Franchise from './components/franchise.hl'
57. import FranchiseEdit from './components/franchiseedit.hl'
58. import TimelineEdit from './components/timelineedit.hl'
59. import { dark, darker } from './shared/tokens.hl'
60. import { postersDir, profilesDir } from './lib/util.hl'
61. import { loginCallbackOf } from './lib/users.hl'
62. import { imageResponse, isSafePosterName } from './lib/images.hl'
63. import { ensureIndex } from './lib/search.hl'
64. import { showsMovedPath, warmTitleCaches, warmEpisodeKeys, movedSlugs, movedPathOf, personById } from './lib/shows.hl'
65. import { allWatchedEpisodeIds } from './lib/watches.hl'
66. import { allFollowedShowIds } from './lib/follows.hl'
67. import { ensureShortIds, shortIdPath } from './lib/shortids.hl'
68. import { guestPersonId, warmPeopleView, peopleViewStats } from './lib/people.hl'
69. import { moveCopiedSummaries } from './lib/dailysync.hl'
70. import { ensureFranchises } from './lib/franchises.hl'
71. import Jobs from './lib/jobs.hl'
73. static appTitle = "tracker"
75. // ---- THE INSTALLABLE APP (tracker.worldapi.org#10), the same way calendar.worldapi.org does it: hl:web's own web app
76. // manifest (/__hl/manifest.webmanifest, linked from every head with the apple-touch-icon and theme-color) and service
77. // worker (/__hl/sw.js) from these settings — no JavaScript of ours. Icons in icons/ (icon.svg is the source, the PNGs
78. // are rendered from it with rsvg-convert, README "Icons"). The theme colour is the header's background (darker), the
79. // splash background the page's (dark) — both off the tokens, not written out again.
80. appThemeColor = darker.value
81. appBackgroundColor = dark.value
82. appIcons = [
83. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'any' }
84. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'any' }
85. { src = '/icons/icon-192.png' sizes = '192x192' purpose = 'maskable' }
86. { src = '/icons/icon-512.png' sizes = '512x512' purpose = 'maskable' }
87. ]
88. appTouchIcon = '/icons/apple-touch-icon.png'
89. appFavicon = '/icons/favicon.svg'
90. // OFFLINE: only the shell — the header and `/` (components/main.hl says "You are offline" while the browser has no
91. // network). The data pages are not kept: without a network they get hl:web's "Unavailable offline" page.
92. offline = [ Home ]
94. styles = Styles
96. port = env('TRACKER_PORT') != null ? toNumber(env('TRACKER_PORT')) : 45008
97. watching = env('TRACKER_WATCH') != '0'
98. // read by hl:web's own manifest config (WebFramework.hl `cfg.sessionDir`), not the constructor call below
99. sessionDir = env('TRACKER_SESSIONS') != null ? env('TRACKER_SESSIONS') : null
101. // ---- the function routes: each checks its input, asks the topic, answers ---------------------------------------------------
102. // /login/callback?ident_code=… (lib/users.hl): the code exchanged, this app's session signed in, back to the page (`?next=`)
103. loginCallback = (&route, &req) => { return loginCallbackOf(server.sessions, req) }
104. // /posters/<name>, /profiles/<name> (lib/images.hl): the file, else the placeholder
105. posterRoute = (&route, &req) => { return imageResponse(postersDir, route.params != null ? route.params.name : null, 'poster') }
106. profileRoute = (&route, &req) => { return imageResponse(profilesDir, route.params != null ? route.params.name : null, 'photo') }
108. // the old addresses of the personal lists (tracker.worldapi.org#8): they all live under /my/ now — a permanent redirect so old
109. // links keep working (architect's decision)
110. movedTo = (&path) => { return new Response('moved to ' + path, { status = 301 headers = { 'Location' = path 'Content-Type' = 'text/plain; charset=utf-8' } }) }
111. notFoundText = (&text) => { return new Response(text, { status = 404 headers = { 'Content-Type' = 'text/plain; charset=utf-8' } }) }
112. unwatchedMoved = (&route, &req) => { return movedTo('/my/unwatched') }
113. scheduleMoved = (&route, &req) => { return movedTo('/my/schedule') }
114. // mission 058: the show page's genre pills linked `/genre/<x>` (no page there) — the genre pages are `/genres/<x>`
115. genreMoved = (&route, &req) => {
116. g = route.params != null ? route.params.genre : null
117. return movedTo(isSafePosterName(g) ? '/genres/' + g : '/')
118. }
119. // mission 060 (tracker.worldapi.org#27): `/<shortId>` → 301 to the title's or person's page (lib/shortids.hl); upper case is
120. // read as lower case; anything else is a 404
121. shortIdRoute = (&route, &req) => {
122. sid = route.params != null ? route.params.sid : null
123. path = sid != null && hlTypeName(sid) == 'String' ? shortIdPath(sid.toLowerCase()) : null
124. if (path == null) { return notFoundText('not found') }
125. return movedTo(path)
126. }
127. // mission 062: a GUEST STAR's link (components/show.hl castRowsOf) — `/person/tmdb/<tmdbId>?show=<show id>` creates the person
128. // (lib/people.hl guestPersonId) or finds the one we have, then 302 → `/person/<slug>`. A function route: the browser loads it
129. // itself (hl:web's client router hands every non-component link to the browser)
130. guestRoute = (&route, &req) => {
131. t = route.params != null ? route.params.tmdb : null
132. tmdbId = t != null && hlTypeName(t) == 'String' && t.length <= 10 ? toNumber(t) : null
133. if (tmdbId == null || tmdbId < 1) { return notFoundText('not found') }
134. q = req.query != null ? req.query : {}
135. r = guestPersonId(tmdbId, q.show)
136. p = r.id != null ? personById(r.id) : null
137. if (p == null || p.urlSegment == null) { return notFoundText('not found: ' + (r.error != null ? r.error : 'no such person')) }
138. return new Response('see /person/' + p.urlSegment, { status = 302 headers = { 'Location' = '/person/' + p.urlSegment 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
139. }
140. // tracker#32 (mission 031): the header's search field sends GET /find?q=<text> (the theme's AppHeader form); a page cannot read a
141. // query string, so this answers 302 → /search/<text> ('/', '?', '#' as spaces, like components/search.hl), no text → /search
142. findRoute = (&route, &req) => {
143. q = req.query != null && req.query.q != null && hlTypeName(req.query.q) == 'String' ? req.query.q.replaceAll('/', ' ').replaceAll('?', ' ').replaceAll('#', ' ').trim() : ''
144. to = q != '' ? '/search/' + encodeURIComponent(q) : '/search'
145. return new Response('see ' + to, { status = 302 headers = { 'Location' = to 'Cache-Control' = 'no-store' 'Content-Type' = 'text/plain; charset=utf-8' } })
146. }
147. // /shows/<slug> (tracker.worldapi.org#21, mission 068): a series or movie (or a merged duplicate's slug) answers 301 to its page
148. // (lib/shows.hl showsMovedPath). hl:web has no way for a component route to answer a redirect, so this is a FUNCTION route, and
149. // for a show (or an unknown slug: the page's own "Not found") it renders the title page the way hl:web renders a component
150. // route (WebFramework.hl `on http.request`, the `component` branch): the Show component matched at /series/<slug> — same
151. // component, same params — with the cookie's session, a new one (and its cookie) when the browser brought none. In the browser
152. // a link to /shows/<slug> is then a full page load; every other title page stays a client-side navigation.
153. showsRoute = (&route, &req) => {
154. slug = route.params != null ? route.params.slug : null
155. to = showsMovedPath(slug)
156. if (to != null) { return movedTo(to) }
157. m = server.router.match('/series/' + slug)
158. let s = req.session
159. fresh = s == null
160. if (fresh) { s = server.sessions.mint() }
161. let res = server.page(m, s, server.hostOf(req.headers['host']), server.requestHeaders(req.headers))
162. if (fresh) { res.headers['Set-Cookie'] = server.sessions.cookieHeader(s.id) }
163. return res
164. }
165. // tracker#31 (mission 026): the slug of a merged duplicate, or a keeper's former slug → 301 to the keeper's page (lib/shows.hl
166. // movedSlugs, read at boot). A component route cannot redirect, so each such slug gets its own function route under /movies/ and
167. // /series/ (/shows/ is showsRoute). A title merged while this process runs answers the keeper's page under its old slug (shows.hl
168. // showBySlug) until the next start.
169. slugMoved = (&route, &req) => {
170. to = movedPathOf(req.path)
171. if (to == null) { return notFoundText('not found') }
172. return movedTo(to)
173. }
175. routes = [
176. { pattern = "/favicon.ico" file = "./icons/favicon.ico" headers = { 'Cache-Control' = 'no-cache' } }
177. // the installable app (tracker.worldapi.org#10): the icons (manifest and service worker are hl:web's own, from appIcons / offline)
178. { pattern = "/icons/icon-192.png" file = "./icons/icon-192.png" headers = { 'Cache-Control' = 'no-cache' } }
179. { pattern = "/icons/icon-512.png" file = "./icons/icon-512.png" headers = { 'Cache-Control' = 'no-cache' } }
180. { pattern = "/icons/apple-touch-icon.png" file = "./icons/apple-touch-icon.png" headers = { 'Cache-Control' = 'no-cache' } }
181. { pattern = "/icons/favicon.svg" file = "./icons/favicon.svg" headers = { 'Cache-Control' = 'no-cache' } }
182. { pattern = "/login/callback" function = loginCallback }
183. { pattern = "/login/failed" component = LoginFailed }
184. { pattern = "/login.js" file = "./login.js" headers = { 'Cache-Control' = 'no-cache' } }
185. { pattern = "/posters/:name" function = posterRoute }
186. // tracker.worldapi.org#13: every movie / every series, 24 per page — the page is a PATH segment (`/movies/page/2`), a
187. // page component cannot read `?page=` (hybriel#11) — and the movies the user follows
188. { pattern = "/movies" component = Movies }
189. { pattern = "/movies/page/:page" component = Movies }
190. // tracker.worldapi.org#21 (mission 068): series and shows split — /series lists the series, /shows the shows; a title's page
191. // is /movies/<slug>, /series/<slug> or /shows/<slug> (one component; /shows/<slug> is showsRoute above)
192. { pattern = "/movies/:slug" component = Show }
193. { pattern = "/series" component = AllSeries }
194. { pattern = "/series/page/:page" component = AllSeries }
195. { pattern = "/series/:slug" component = Show }
196. { pattern = "/shows" component = AllShows }
197. { pattern = "/shows/page/:page" component = AllShows }
198. { pattern = "/shows/:slug" function = showsRoute }
199. { pattern = "/my/movies" component = MyMovies }
200. { pattern = "/my/unwatched" component = Unwatched }
201. { pattern = "/my/schedule" component = Schedule }
202. // #21: /my/series and /my/shows (one component, components/myshows.hl; any other word there is its "Not found")
203. { pattern = "/my/:tv" component = MyShows }
204. { pattern = "/unwatched" function = unwatchedMoved }
205. { pattern = "/schedule" function = scheduleMoved }
206. // tracker.worldapi.org#14: the search — `/search/<text>` (hl:web gives a page no query string, so not `?q=`)
207. { pattern = "/search/:q?" component = Search }
208. { pattern = "/find" function = findRoute }
209. // tracker.worldapi.org#16: a person's page (the show page's cast and the search link here) and their photo
210. { pattern = "/person/tmdb/:tmdb" function = guestRoute }
211. { pattern = "/person/:slug" component = Person }
212. { pattern = "/profiles/:name" function = profileRoute }
213. // mission 058 (tracker.worldapi.org#25): a genre's movies and series, like /movies (the page in the path)
214. { pattern = "/genres/:genre" component = Genre }
215. { pattern = "/genres/:genre/page/:page" component = Genre }
216. { pattern = "/genre/:genre" function = genreMoved }
217. // tracker#32 (mission 031): the main menu's Genres and People, the user menu's Settings
218. { pattern = "/genres" component = Genres }
219. // tracker#37 (mission 032): /people = everyone, the one updated last first, 24 per page (the page in the path, like /movies)
220. { pattern = "/people" component = People }
221. { pattern = "/people/page/:page" component = People }
222. { pattern = "/settings" component = Settings }
223. // tracker.worldapi.org#19 (mission 066): franchises and timelines, the creator's editor (`/franchises/edit` before the slug)
224. { pattern = "/franchises" component = Franchises }
225. { pattern = "/franchises/edit" component = FranchiseEdit }
226. { pattern = "/franchises/:slug" component = Franchise }
227. { pattern = "/timelines/:tslug" component = Franchise }
228. { pattern = "/timelines/:slug/edit" component = TimelineEdit }
229. { pattern = "/" component = Home }
230. // mission 060 (tracker.worldapi.org#27): a short id → its page (LAST: every named one-segment route above wins)
231. { pattern = "/:sid" function = shortIdRoute }
232. ]
234. // WHO GETS THE PUSH: the login state reaches the tabs of one session (components/main.hl trackerSignedIn/Out).
235. tagOf = (&session) => { return session != null && session.data != null ? session.data.tag : null }
236. audience = {
237. trackerSignedIn = (tag, &session) => { return tag != null && tagOf(session) == tag }
238. trackerSignedOut = (tag, &session) => { return tag != null && tagOf(session) == tag }
239. }
241. // cookies are per host, not per port: an own name keeps this app's session apart from
242. // ident's own (and from any other worldapi app sharing a dev host), see ident README "Design
243. // tokens" / hybriel#10 hybriel#17.
244. sessionCookie = 'trackersid'
245. // tracker.worldapi.org#14: the search index (lib/search.hl) is built once here at boot (~0.6 s on the real data), not by the
246. // first search
247. searchIndexReady = ensureIndex()
248. // mission 060: every short id in use (one map over shows AND persons) and the records still without one, before the first request
249. shortIdsReady = ensureShortIds()
250. // tracker#30 (mission 025): the slim list caches (lib/shows.hl cardOf / episodeRowsOf) of every followed title, before the first
251. // /my/ load — not filled by that load
252. titleCachesStart = now()
253. titleCachesWarm = warmTitleCaches(allFollowedShowIds())
254. episodeKeysWarm = warmEpisodeKeys(allWatchedEpisodeIds())
255. console.log('list caches: ' + titleCachesWarm + ' followed titles, ' + episodeKeysWarm + ' watched episodes in ' + (now() - titleCachesStart) + ' ms')
256. // tracker#37 (mission 032, lib/people.hl): /people's order (everyone, the one updated last first) and the photos from the titles'
257. // casts, before the first /people load — not built by that load
258. peopleViewStart = now()
259. peopleViewWarm = warmPeopleView()
260. console.log('people view: ' + peopleViewWarm + ' people, ' + peopleViewStats().castProfiles + ' cast photos in ' + (now() - peopleViewStart) + ' ms')
261. // tracker#18, ONCE (lib/dailysync.hl moveCopiedSummaries): a `summary` that is only the migrated copy of tmdbSummary is MOVED to
262. // `migratedSummary` (mission 070: kept, not shown) — the current TMDB text shows; the creator's own texts stay
263. summariesMoved = moveCopiedSummaries()
264. if (summariesMoved.ran) { console.log('summaries: moved ' + summariesMoved.moved + ' copies of tmdbSummary to migratedSummary, kept ' + summariesMoved.kept + ' own texts, errors ' + summariesMoved.errors + ', ' + summariesMoved.ms + ' ms') }
265. // mission 066: which timelines every title is in (one scan of the timelines), before the first show page
266. franchisesReady = ensureFranchises()
267. // tracker#31: the moved slugs' routes first, then `routes` as written (hl:web reads the written list for the pages it serves)
268. movedSlugList = movedSlugs()
269. withMovedSlugs = () => {
270. out = []
271. for (m of movedSlugList) {
272. out.push({ pattern = '/movies/' + m.slug function = slugMoved })
273. out.push({ pattern = '/series/' + m.slug function = slugMoved })
274. }
275. for (r of routes) { out.push(r) }
276. return out
277. }
278. allRoutes = withMovedSlugs()
279. console.log('moved slugs: ' + movedSlugList.length + ' (301 to the keeper)')
280. server = new WebFramework(routes = allRoutes, styles = styles, port = port, sessionCookie = sessionCookie, watchMode = watching)
282. // ---- THE BACKGROUND JOBS (lib/jobs.hl): the daily TMDB sync, the backfills, the details repair, the credits job, the duplicate
283. // merge, the short-id backfill, the collection seed, the curated franchise seed — constructing it starts their clocks and logs which ones are off
284. jobs = new Jobs()
286. on Error(e) { console.log('error absorbed: ' + e.message) }