diff --git a/package.json b/package.json index 8a5e3dcc11..5a68c7a65b 100644 --- a/package.json +++ b/package.json @@ -68,11 +68,11 @@ "humanize-duration": "^3.33.2", "js-yaml": "^4.1.1", "lodash": "4.18.1", + "lru-cache": "11.2.1", "mime": "^4.1.0", "nanoid": "^5.1.7", "next": "16.2.6", "ip-address": "^10.2.0", - "node-cache": "5.1.2", "node-gyp": "12.2.0", "node-schedule": "2.1.1", "nodemailer": "8.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0f33b8a92a..a168e256cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: lodash: specifier: 4.18.1 version: 4.18.1 + lru-cache: + specifier: 11.2.1 + version: 11.2.1 mime: specifier: ^4.1.0 version: 4.1.0 @@ -128,9 +131,6 @@ importers: next: specifier: 16.2.6 version: 16.2.6(@babel/core@7.29.0)(babel-plugin-macros@3.1.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - node-cache: - specifier: 5.1.2 - version: 5.1.2 node-gyp: specifier: 12.2.0 version: 12.2.0 @@ -3425,10 +3425,6 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} - clone@2.1.2: - resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} - engines: {node: '>=0.8'} - clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} @@ -5939,10 +5935,6 @@ packages: resolution: {integrity: sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==} engines: {node: ^18 || ^20 || >= 21} - node-cache@5.1.2: - resolution: {integrity: sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==} - engines: {node: '>= 8.0.0'} - node-exports-info@1.6.0: resolution: {integrity: sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==} engines: {node: '>= 0.4'} @@ -11429,8 +11421,6 @@ snapshots: clone@1.0.4: {} - clone@2.1.2: {} - clsx@2.1.1: {} co@4.6.0: @@ -14707,10 +14697,6 @@ snapshots: node-addon-api@8.5.0: {} - node-cache@5.1.2: - dependencies: - clone: 2.1.2 - node-exports-info@1.6.0: dependencies: array.prototype.flatmap: 1.3.3 diff --git a/server/api/externalapi.ts b/server/api/externalapi.ts index 9b71b7a159..b2b8c4463f 100644 --- a/server/api/externalapi.ts +++ b/server/api/externalapi.ts @@ -1,8 +1,8 @@ +import type { CacheStore } from '@server/lib/cache'; import { proxyRequestInterceptor } from '@server/utils/customProxyAgent'; import type { AxiosInstance, AxiosRequestConfig } from 'axios'; import axios from 'axios'; import rateLimit from 'axios-rate-limit'; -import type NodeCache from 'node-cache'; // 5 minute default TTL (in seconds) const DEFAULT_TTL = 300; @@ -11,7 +11,7 @@ const DEFAULT_TTL = 300; const DEFAULT_ROLLING_BUFFER = 10000; export interface ExternalAPIOptions { - nodeCache?: NodeCache; + nodeCache?: CacheStore; headers?: Record; timeout?: number; rateLimit?: { @@ -23,7 +23,7 @@ export interface ExternalAPIOptions { class ExternalAPI { protected axios: AxiosInstance; private baseUrl: string; - private cache?: NodeCache; + private cache?: CacheStore; constructor( baseUrl: string, @@ -53,27 +53,33 @@ class ExternalAPI { this.cache = options.nodeCache; } + // transform runs before the cache write. protected async get( endpoint: string, config?: AxiosRequestConfig, - ttl?: number + ttl?: number, + options?: { cache?: CacheStore; transform?: (data: T) => T } ): Promise { + const cache = options?.cache ?? this.cache; const cacheKey = this.serializeCacheKey(endpoint, { ...config?.params, headers: config?.headers, }); - const cachedItem = this.cache?.get(cacheKey); + const cachedItem = cache?.get(cacheKey); if (cachedItem) { return cachedItem; } const response = await this.axios.get(endpoint, config); + const data = options?.transform + ? options.transform(response.data) + : response.data; - if (this.cache && ttl !== 0) { - this.cache.set(cacheKey, response.data, ttl ?? DEFAULT_TTL); + if (cache && ttl !== 0) { + cache.set(cacheKey, data, ttl ?? DEFAULT_TTL); } - return response.data; + return data; } protected async post( diff --git a/server/api/plexapi.ts b/server/api/plexapi.ts index 81e016717f..0837569b03 100644 --- a/server/api/plexapi.ts +++ b/server/api/plexapi.ts @@ -228,8 +228,9 @@ class PlexAPI extends ExternalAPI { mediaType: 'movie' | 'show' ): Promise { const response = await this.get( - `/library/sections/${id}/all?type=${ - mediaType === 'show' ? '4' : '1' + `/library/sections/${id}/all?type=${mediaType === 'show' ? '4' : '1'}${ + // Shows are queried as episodes, whose guids the scanner never reads. + mediaType === 'movie' ? '&includeGuids=1' : '' }&sort=addedAt%3Adesc&addedAt>>=${Math.floor(options.addedAt / 1000)}`, { headers: { diff --git a/server/api/themoviedb/index.ts b/server/api/themoviedb/index.ts index 305d6048d7..5e4807f48f 100644 --- a/server/api/themoviedb/index.ts +++ b/server/api/themoviedb/index.ts @@ -1,5 +1,6 @@ import ExternalAPI from '@server/api/externalapi'; import type { TvShowProvider } from '@server/api/provider'; +import type { CacheStore } from '@server/lib/cache'; import cacheManager from '@server/lib/cache'; import { getSettings } from '@server/lib/settings'; import { sortBy } from 'lodash'; @@ -23,6 +24,7 @@ import type { TmdbSearchTvResponse, TmdbSeasonWithEpisodes, TmdbTvDetails, + TmdbTvScanDetails, TmdbUpcomingMoviesResponse, TmdbWatchProviderDetails, TmdbWatchProviderRegion, @@ -124,7 +126,43 @@ interface DiscoverTvOptions { certificationCountry?: string; } +const TV_DETAILS_TTL = 43200; + +// Scan lookups read an entry once and never again. +const SCAN_TTL = 900; + +const TV_DETAILS_APPEND_TO_RESPONSE = + 'aggregate_credits,credits,external_ids,keywords,videos,content_ratings,watch/providers'; + +const TV_SCAN_APPEND_TO_RESPONSE = 'keywords,external_ids'; + +type ExternalIdLookup = + | { + externalId: string; + type: 'imdb'; + language?: string; + } + | { + externalId: number; + type: 'tvdb'; + language?: string; + }; + +// Nothing anywhere reads aggregate_credits.crew or credits.cast. +const stripUnreadTvCredits = (data: T): T => { + const show = data as { + aggregate_credits?: { crew?: unknown }; + credits?: { cast?: unknown }; + }; + + delete show.aggregate_credits?.crew; + delete show.credits?.cast; + + return data; +}; + class TheMovieDb extends ExternalAPI implements TvShowProvider { + private scanCache = cacheManager.getCache('tmdbscan').data; private locale: string; private discoverRegion?: string; private originalLanguage?: string; @@ -341,6 +379,37 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { } }; + // The append list is part of the cache key, so full and narrow entries differ. + private getTvShowDetails = async ({ + tvId, + language, + appendToResponse, + includeVideoLanguage, + ttl, + cache, + }: { + tvId: number; + language?: string; + appendToResponse: string; + includeVideoLanguage?: string; + ttl: number; + cache?: CacheStore; + }): Promise => + this.get( + `/tv/${tvId}`, + { + params: { + language, + append_to_response: appendToResponse, + ...(includeVideoLanguage + ? { include_video_language: includeVideoLanguage } + : {}), + }, + }, + ttl, + { cache, transform: stripUnreadTvCredits } + ); + public getTvShow = async ({ tvId, language = this.locale, @@ -349,35 +418,26 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { language?: string; }): Promise => { try { - const data = await this.get( - `/tv/${tvId}`, - { - params: { - language, - append_to_response: - 'aggregate_credits,credits,external_ids,keywords,videos,content_ratings,watch/providers', - include_video_language: language, - }, - }, - 43200 - ); + const data = await this.getTvShowDetails({ + tvId, + language, + appendToResponse: TV_DETAILS_APPEND_TO_RESPONSE, + includeVideoLanguage: language, + ttl: TV_DETAILS_TTL, + }); if ( (!language || !language.startsWith('en')) && !data.videos?.results?.some((video) => video.type === 'Trailer') ) { try { - const fallback = await this.get( - `/tv/${tvId}`, - { - params: { - language, - append_to_response: 'videos', - include_video_language: 'en', - }, - }, - 43200 - ); + const fallback = await this.getTvShowDetails({ + tvId, + language, + appendToResponse: 'videos', + includeVideoLanguage: 'en', + ttl: TV_DETAILS_TTL, + }); const localizedVideos = data.videos?.results ?? []; const localizedVideoKeys = new Set( @@ -408,6 +468,29 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { } }; + public getTvShowForScan = async ({ + tvId, + language = this.locale, + }: { + tvId: number; + language?: string; + }): Promise => { + try { + return await this.getTvShowDetails({ + tvId, + language, + appendToResponse: TV_SCAN_APPEND_TO_RESPONSE, + ttl: SCAN_TTL, + cache: this.scanCache, + }); + } catch (e) { + throw new Error( + `[TMDB] Failed to fetch TV show scan details: ${e.message}`, + { cause: e } + ); + } + }; + public getTvSeason = async ({ tvId, seasonNumber, @@ -871,33 +954,36 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { } }; - public async getByExternalId({ - externalId, - type, - language = this.locale, - }: - | { - externalId: string; - type: 'imdb'; - language?: string; - } - | { - externalId: number; - type: 'tvdb'; - language?: string; - }): Promise { + public async getByExternalId( + lookup: ExternalIdLookup + ): Promise { + return this.findByExternalId(lookup); + } + + public async getByExternalIdForScan( + lookup: ExternalIdLookup + ): Promise { + return this.findByExternalId(lookup, this.scanCache, SCAN_TTL); + } + + // Omitting ttl here silently falls back to DEFAULT_TTL, regardless of tier. + private async findByExternalId( + { externalId, type, language = this.locale }: ExternalIdLookup, + cache?: CacheStore, + ttl?: number + ): Promise { try { - const data = await this.get( + return await this.get( `/find/${externalId}`, { params: { external_source: type === 'imdb' ? 'imdb_id' : 'tvdb_id', language, }, - } + }, + ttl, + { cache } ); - - return data; } catch (e) { throw new Error(`[TMDB] Failed to find by external ID: ${e.message}`, { cause: e, @@ -905,35 +991,31 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { } } - public async getMediaByImdbId({ + // A /find hit can be stale, so confirm the matched id still exists. + public async resolveImdbIdForScan({ imdbId, - language = this.locale, }: { imdbId: string; - language?: string; - }): Promise { + }): Promise { try { - const extResponse = await this.getByExternalId({ - externalId: imdbId, - type: 'imdb', - }); + const extResponse = await this.findByExternalId( + { externalId: imdbId, type: 'imdb' }, + this.scanCache, + SCAN_TTL + ); if (extResponse.movie_results[0]) { - const movie = await this.getMovie({ - movieId: extResponse.movie_results[0].id, - language, - }); - - return movie; + return await this.assertMovieExistsForScan( + extResponse.movie_results[0].id + ); } if (extResponse.tv_results[0]) { - const tvshow = await this.getTvShow({ + const tvShow = await this.getTvShowForScan({ tvId: extResponse.tv_results[0].id, - language, }); - return tvshow; + return tvShow.id; } throw new Error(`No movie or show returned from API for ID ${imdbId}`); @@ -945,6 +1027,35 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { } } + private async assertMovieExistsForScan(movieId: number): Promise { + const data = await this.get<{ id: number }>( + `/movie/${movieId}`, + {}, + SCAN_TTL, + { cache: this.scanCache } + ); + + return data.id; + } + + private async resolveTvdbId( + tvdbId: number, + cache?: CacheStore, + ttl?: number + ): Promise { + const extResponse = await this.findByExternalId( + { externalId: tvdbId, type: 'tvdb' }, + cache, + ttl + ); + + if (!extResponse.tv_results[0]) { + throw new Error(`No show returned from API for ID ${tvdbId}`); + } + + return extResponse.tv_results[0].id; + } + public async getShowByTvdbId({ tvdbId, language = this.locale, @@ -953,21 +1064,30 @@ class TheMovieDb extends ExternalAPI implements TvShowProvider { language?: string; }): Promise { try { - const extResponse = await this.getByExternalId({ - externalId: tvdbId, - type: 'tvdb', + return await this.getTvShow({ + tvId: await this.resolveTvdbId(tvdbId), + language, }); + } catch (e) { + throw new Error( + `[TMDB] Failed to get TV show using the external TVDB ID: ${e.message}`, + { cause: e } + ); + } + } - if (extResponse.tv_results[0]) { - const tvshow = await this.getTvShow({ - tvId: extResponse.tv_results[0].id, - language, - }); - - return tvshow; - } - - throw new Error(`No show returned from API for ID ${tvdbId}`); + public async getShowByTvdbIdForScan({ + tvdbId, + language = this.locale, + }: { + tvdbId: number; + language?: string; + }): Promise { + try { + return await this.getTvShowForScan({ + tvId: await this.resolveTvdbId(tvdbId, this.scanCache, SCAN_TTL), + language, + }); } catch (e) { throw new Error( `[TMDB] Failed to get TV show using the external TVDB ID: ${e.message}`, diff --git a/server/api/themoviedb/interfaces.ts b/server/api/themoviedb/interfaces.ts index a4c49e9ab0..cd68ccc70b 100644 --- a/server/api/themoviedb/interfaces.ts +++ b/server/api/themoviedb/interfaces.ts @@ -304,6 +304,16 @@ export interface TmdbTvDetails { }; } +export interface TmdbTvScanDetails { + id: number; + name: string; + seasons: TmdbTvSeasonResult[]; + external_ids: TmdbExternalIds; + keywords: { + results: TmdbKeyword[]; + }; +} + export interface TmdbVideoResult { results: TmdbVideo[]; } diff --git a/server/lib/availabilitySync.test.ts b/server/lib/availabilitySync.test.ts index 4bb68e483e..45c3f50f54 100644 --- a/server/lib/availabilitySync.test.ts +++ b/server/lib/availabilitySync.test.ts @@ -152,7 +152,7 @@ let getShowByTvdbIdImpl: (args: { language?: string; }) => Promise = async () => fakeTmdbShow(1); -Object.defineProperty(TheMovieDb.prototype, 'getTvShow', { +Object.defineProperty(TheMovieDb.prototype, 'getTvShowForScan', { get() { return async (args: { tvId: number; language?: string }) => getTvShowImpl(args); @@ -161,7 +161,7 @@ Object.defineProperty(TheMovieDb.prototype, 'getTvShow', { configurable: true, }); -Object.defineProperty(TheMovieDb.prototype, 'getShowByTvdbId', { +Object.defineProperty(TheMovieDb.prototype, 'getShowByTvdbIdForScan', { get() { return async (args: { tvdbId: number; language?: string }) => getShowByTvdbIdImpl(args); diff --git a/server/lib/availabilitySync.ts b/server/lib/availabilitySync.ts index 1636744ad1..4a700ef355 100644 --- a/server/lib/availabilitySync.ts +++ b/server/lib/availabilitySync.ts @@ -6,7 +6,10 @@ import RadarrAPI, { type RadarrMovie } from '@server/api/servarr/radarr'; import type { SonarrSeason, SonarrSeries } from '@server/api/servarr/sonarr'; import SonarrAPI from '@server/api/servarr/sonarr'; import TheMovieDb from '@server/api/themoviedb'; -import type { TmdbTvDetails } from '@server/api/themoviedb/interfaces'; +import type { + TmdbTvDetails, + TmdbTvScanDetails, +} from '@server/api/themoviedb/interfaces'; import { MediaRequestStatus, MediaStatus } from '@server/constants/media'; import { MediaServerType } from '@server/constants/server'; import { getRepository } from '@server/datasource'; @@ -362,14 +365,14 @@ class AvailabilitySync { } // We need to fetch from TMDB to get the episode count for each season - let tvShow: TmdbTvDetails | undefined; + let tvShow: TmdbTvScanDetails | TmdbTvDetails | undefined; try { if (media.tmdbId) { - tvShow = await this.tmdb.getTvShow({ + tvShow = await this.tmdb.getTvShowForScan({ tvId: Number(media.tmdbId), }); } else if (media.tvdbId) { - tvShow = await this.tmdb.getShowByTvdbId({ + tvShow = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: Number(media.tvdbId), }); } diff --git a/server/lib/cache.ts b/server/lib/cache.ts index 64b5c79ee8..914cafbb52 100644 --- a/server/lib/cache.ts +++ b/server/lib/cache.ts @@ -1,35 +1,208 @@ -import NodeCache from 'node-cache'; +import { LRUCache } from 'lru-cache'; export type AvailableCacheIds = | 'tmdb' + | 'tmdbscan' | 'radarr' | 'sonarr' | 'rt' | 'imdb' | 'github' - | 'plexguid' | 'plextv' | 'plexwatchlist' | 'tvdb'; const DEFAULT_TTL = 300; -const DEFAULT_CHECK_PERIOD = 120; + +const OBJECT_VALUE_SIZE = 80; +const PROMISE_VALUE_SIZE = 80; +const ARRAY_VALUE_SIZE = 40; + +// Limits key count, not memory, since TMDB never caps the credits rosters. +const TMDB_MAX_KEYS = 1000; + +// Backstop against unbounded growth as the 15 minute TTL is what bounds this tier. +const TMDB_SCAN_MAX_KEYS = 2000; + +// Only the rolling quality profile, root folder and language profile lookups reach +// these tiers, and keys are prefixed by server url, so this is a few keys per server. +const RADARR_MAX_KEYS = 64; +const SONARR_MAX_KEYS = 64; + +// One key per searched title, each holding twenty search hits. +const RT_MAX_KEYS = 500; + +// One key per IMDb id, requested from the same endpoint as the RT ratings. +const IMDB_MAX_KEYS = 500; + +// Releases and commits, varying only by page size and branch. +const GITHUB_MAX_KEYS = 16; + +// Watchlist item metadata, shared between users as the token is not part of the key. +const PLEX_TV_MAX_KEYS = 5000; + +// Keyed by auth token, so one key per Plex linked user. +const PLEX_WATCHLIST_MAX_KEYS = 500; + +// Several keys per show, holding the largest payloads of any tier as the extended +// series lookup carries every episode. +const TVDB_MAX_KEYS = 500; + +export interface CacheStats { + hits: number; + misses: number; + keys: number; + ksize: number; + vsize: number; +} + +export interface CacheStore { + get(key: string): T | undefined; + set(key: string, value: T, ttl?: number): boolean; + del(key: string): number; + getTtl(key: string): number | undefined; + getStats(): CacheStats; + flushAll(): void; +} + +const keyLength = (key: string): number => key.toString().length; + +const valueLength = (value: unknown): number => { + if (typeof value === 'string') { + return value.length; + } + if (Array.isArray(value)) { + return ARRAY_VALUE_SIZE * value.length; + } + if (typeof value === 'number') { + return 8; + } + if ( + typeof (value as { then?: unknown } | null | undefined)?.then === 'function' + ) { + return PROMISE_VALUE_SIZE; + } + if (Buffer.isBuffer(value)) { + return value.length; + } + if (value != null && typeof value === 'object') { + return OBJECT_VALUE_SIZE * Object.keys(value).length; + } + if (typeof value === 'boolean') { + return 8; + } + return 0; +}; + +interface LruEntry { + value: unknown; + ksize: number; + vsize: number; +} + +class LruCacheStore implements CacheStore { + private cache: LRUCache; + private stdTtl: number; + private hits = 0; + private misses = 0; + private ksize = 0; + private vsize = 0; + + constructor({ max, stdTtl }: { max: number; stdTtl: number }) { + this.stdTtl = stdTtl; + this.cache = new LRUCache({ + max, + ttl: stdTtl * 1000, + // Without this, expired entries are only dropped when their key is read. + ttlAutopurge: true, + // Fires on every removal path, so one decrement here covers all of them. + dispose: (entry) => { + this.ksize -= entry.ksize; + this.vsize -= entry.vsize; + }, + }); + } + + // Cloned on read because callers mutate what they get back, as getTvSeason + // does when it rewrites still_path in place. + public get(key: string): T | undefined { + const entry = this.cache.get(key); + + if (entry === undefined) { + this.misses++; + return undefined; + } + + this.hits++; + return structuredClone(entry.value) as T; + } + + public set(key: string, value: T, ttl?: number): boolean { + const entry: LruEntry = { + value: structuredClone(value), + ksize: keyLength(key), + vsize: valueLength(value), + }; + + // A ttl of 0 means the entry never expires. + this.cache.set(key, entry, { ttl: (ttl ?? this.stdTtl) * 1000 }); + + this.ksize += entry.ksize; + this.vsize += entry.vsize; + + return true; + } + + public del(key: string): number { + return this.cache.delete(key) ? 1 : 0; + } + + public getTtl(key: string): number | undefined { + if (!this.cache.has(key)) { + return undefined; + } + + const remaining = this.cache.getRemainingTTL(key); + + // Keys with no expiry report 0. + return remaining === Infinity ? 0 : Date.now() + remaining; + } + + public getStats(): CacheStats { + return { + hits: this.hits, + misses: this.misses, + keys: this.cache.size, + ksize: this.ksize, + vsize: this.vsize, + }; + } + + public flushAll(): void { + this.cache.clear(); + this.hits = 0; + this.misses = 0; + this.ksize = 0; + this.vsize = 0; + } +} class Cache { public id: AvailableCacheIds; - public data: NodeCache; + public data: CacheStore; public name: string; constructor( id: AvailableCacheIds, name: string, - options: { stdTtl?: number; checkPeriod?: number } = {} + options: { max: number; stdTtl?: number } ) { this.id = id; this.name = name; - this.data = new NodeCache({ - stdTTL: options.stdTtl ?? DEFAULT_TTL, - checkperiod: options.checkPeriod ?? DEFAULT_CHECK_PERIOD, + + this.data = new LruCacheStore({ + max: options.max, + stdTtl: options.stdTtl ?? DEFAULT_TTL, }); } @@ -46,34 +219,36 @@ class CacheManager { private availableCaches: Record = { tmdb: new Cache('tmdb', 'The Movie Database API', { stdTtl: 21600, - checkPeriod: 60 * 30, + max: TMDB_MAX_KEYS, + }), + tmdbscan: new Cache('tmdbscan', 'The Movie Database API (Library Scans)', { + stdTtl: 900, + max: TMDB_SCAN_MAX_KEYS, }), - radarr: new Cache('radarr', 'Radarr API'), - sonarr: new Cache('sonarr', 'Sonarr API'), + radarr: new Cache('radarr', 'Radarr API', { max: RADARR_MAX_KEYS }), + sonarr: new Cache('sonarr', 'Sonarr API', { max: SONARR_MAX_KEYS }), rt: new Cache('rt', 'Rotten Tomatoes API', { stdTtl: 43200, - checkPeriod: 60 * 30, + max: RT_MAX_KEYS, }), imdb: new Cache('imdb', 'IMDB Radarr Proxy', { stdTtl: 43200, - checkPeriod: 60 * 30, + max: IMDB_MAX_KEYS, }), github: new Cache('github', 'GitHub API', { stdTtl: 21600, - checkPeriod: 60 * 30, - }), - plexguid: new Cache('plexguid', 'Plex GUID', { - stdTtl: 86400 * 7, // 1 week cache - checkPeriod: 60 * 30, + max: GITHUB_MAX_KEYS, }), plextv: new Cache('plextv', 'Plex TV', { stdTtl: 86400 * 7, // 1 week cache - checkPeriod: 60, + max: PLEX_TV_MAX_KEYS, + }), + plexwatchlist: new Cache('plexwatchlist', 'Plex Watchlist', { + max: PLEX_WATCHLIST_MAX_KEYS, }), - plexwatchlist: new Cache('plexwatchlist', 'Plex Watchlist'), tvdb: new Cache('tvdb', 'The TVDB API', { stdTtl: 21600, - checkPeriod: 60 * 30, + max: TVDB_MAX_KEYS, }), }; diff --git a/server/lib/scanners/jellyfin/index.ts b/server/lib/scanners/jellyfin/index.ts index 64faab6268..a3e2300193 100644 --- a/server/lib/scanners/jellyfin/index.ts +++ b/server/lib/scanners/jellyfin/index.ts @@ -10,6 +10,7 @@ import { ANIME_KEYWORD_ID } from '@server/api/themoviedb/constants'; import type { TmdbKeyword, TmdbTvDetails, + TmdbTvScanDetails, } from '@server/api/themoviedb/interfaces'; import { MediaServerType } from '@server/constants/server'; import { getRepository } from '@server/datasource'; @@ -73,10 +74,9 @@ class JellyfinScanner } if (imdbId && !tmdbId) { - const tmdbMovie = await this.tmdb.getMediaByImdbId({ + tmdbId = await this.tmdb.resolveImdbIdForScan({ imdbId: imdbId, }); - tmdbId = tmdbMovie.id; } if (!tmdbId) { @@ -184,15 +184,15 @@ class JellyfinScanner }: { tmdbId?: number; tvdbId?: number; - }): Promise { + }): Promise { let tvShow; if (tmdbId) { - tvShow = await this.tmdb.getTvShow({ + tvShow = await this.tmdb.getTvShowForScan({ tvId: Number(tmdbId), }); } else if (tvdbId) { - tvShow = await this.tmdb.getShowByTvdbId({ + tvShow = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: Number(tvdbId), }); } else { @@ -215,7 +215,7 @@ class JellyfinScanner } private async processJellyfinShow(jellyfinitem: JellyfinLibraryItem) { - let tvShow: TmdbTvDetails | null = null; + let tvShow: TmdbTvScanDetails | TmdbTvDetails | null = null; try { const Id = @@ -262,7 +262,7 @@ class JellyfinScanner tvdbSeasonFromAnidb = result?.tvdbSeason; if (result?.tvdbId) { try { - tvShow = await this.tmdb.getShowByTvdbId({ + tvShow = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: result.tvdbId, }); } catch { diff --git a/server/lib/scanners/jellyfin/jellyfin.test.ts b/server/lib/scanners/jellyfin/jellyfin.test.ts index 51f97d09a8..d56e139d41 100644 --- a/server/lib/scanners/jellyfin/jellyfin.test.ts +++ b/server/lib/scanners/jellyfin/jellyfin.test.ts @@ -99,6 +99,15 @@ Object.defineProperty(TheMovieDb.prototype, 'getTvShow', { configurable: true, }); +Object.defineProperty(TheMovieDb.prototype, 'getTvShowForScan', { + get() { + return async (args: { tvId: number; language?: string }) => + getTvShowImpl(args); + }, + set() {}, + configurable: true, +}); + import { jellyfinFullScanner } from '@server/lib/scanners/jellyfin'; setupTestDb(); diff --git a/server/lib/scanners/plex/index.ts b/server/lib/scanners/plex/index.ts index 37f951d706..f3eda4a3bc 100644 --- a/server/lib/scanners/plex/index.ts +++ b/server/lib/scanners/plex/index.ts @@ -7,10 +7,10 @@ import { ANIME_KEYWORD_ID } from '@server/api/themoviedb/constants'; import type { TmdbKeyword, TmdbTvDetails, + TmdbTvScanDetails, } from '@server/api/themoviedb/interfaces'; import { getRepository } from '@server/datasource'; import { User } from '@server/entity/User'; -import cacheManager from '@server/lib/cache'; import type { MediaIds, ProcessableSeason, @@ -264,15 +264,15 @@ class PlexScanner }: { tmdbId?: number; tvdbId?: number; - }): Promise { + }): Promise { let tvShow; if (tmdbId) { - tvShow = await this.tmdb.getTvShow({ + tvShow = await this.tmdb.getTvShowForScan({ tvId: Number(tmdbId), }); } else if (tvdbId) { - tvShow = await this.tmdb.getShowByTvdbId({ + tvShow = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: Number(tvdbId), }); } else { @@ -383,24 +383,12 @@ class PlexScanner } private async getMediaIds(plexitem: PlexLibraryItem): Promise { - let mediaIds: Partial = {}; + const mediaIds: Partial = {}; // Check if item is using new plex movie/tv agent if ( plexitem.guid.match(plexRegex) || plexitem.guid.match(plexCustomProviderRegex) ) { - const guidCache = cacheManager.getCache('plexguid'); - - const cachedGuids = guidCache.data.get(plexitem.ratingKey); - - if (cachedGuids) { - this.log('GUIDs are cached. Skipping metadata request.', 'debug', { - mediaIds: cachedGuids, - title: plexitem.title, - }); - mediaIds = cachedGuids; - } - const metadata = plexitem.Guid && plexitem.Guid.length > 0 ? plexitem @@ -428,31 +416,26 @@ class PlexScanner // If we got an IMDb ID, but no TMDB ID, lookup the TMDB ID with the IMDb ID if (mediaIds.imdbId && !mediaIds.tmdbId) { - const tmdbMedia = await this.tmdb.getMediaByImdbId({ + mediaIds.tmdbId = await this.tmdb.resolveImdbIdForScan({ imdbId: mediaIds.imdbId, }); - mediaIds.tmdbId = tmdbMedia.id; } if (mediaIds.tvdbId && !mediaIds.tmdbId) { - const show = await this.tmdb.getShowByTvdbId({ + const show = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: mediaIds.tvdbId, }); mediaIds.tmdbId = show.id; } - // Cache GUIDs - guidCache.data.set(plexitem.ratingKey, mediaIds); - // Check if the agent is IMDb } else if (plexitem.guid.match(imdbRegex)) { const imdbMatch = plexitem.guid.match(imdbRegex); if (imdbMatch) { mediaIds.imdbId = imdbMatch[1]; - const tmdbMedia = await this.tmdb.getMediaByImdbId({ + mediaIds.tmdbId = await this.tmdb.resolveImdbIdForScan({ imdbId: mediaIds.imdbId, }); - mediaIds.tmdbId = tmdbMedia.id; } // Check if the agent is TMDB } else if (plexitem.guid.match(tmdbRegex)) { @@ -464,9 +447,9 @@ class PlexScanner } else if (plexitem.guid.match(tvdbRegex)) { const matchedtvdb = plexitem.guid.match(tvdbRegex); - // If we can find a tvdb Id, use it to get the full tmdb show details + // If we can find a tvdb Id, use it to resolve the tmdb id if (matchedtvdb) { - const show = await this.tmdb.getShowByTvdbId({ + const show = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: Number(matchedtvdb[1]), }); @@ -484,7 +467,7 @@ class PlexScanner const matchedtvdb = plexitem.guid.match(hamaTvdbRegex); if (matchedtvdb) { - const show = await this.tmdb.getShowByTvdbId({ + const show = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: Number(matchedtvdb[1]), }); @@ -506,19 +489,19 @@ class PlexScanner } else if (matchedhama) { const anidbId = Number(matchedhama[1]); const result = animeList.getFromAnidbId(anidbId); - let tvShow: TmdbTvDetails | null = null; + let tvShow: TmdbTvScanDetails | TmdbTvDetails | null = null; // Set isHama to true, so we can know to add special processing to this item mediaIds.isHama = true; // First try to lookup the show by TVDb ID if (result?.tvdbId) { - const extResponse = await this.tmdb.getByExternalId({ + const extResponse = await this.tmdb.getByExternalIdForScan({ externalId: result.tvdbId, type: 'tvdb', }); if (extResponse.tv_results[0]) { - tvShow = await this.tmdb.getTvShow({ + tvShow = await this.tmdb.getTvShowForScan({ tvId: extResponse.tv_results[0].id, }); mediaIds.tvdbId = result.tvdbId; @@ -537,10 +520,9 @@ class PlexScanner mediaIds.tmdbId = result.tmdbId; mediaIds.imdbId = result?.imdbId; } else if (result?.imdbId) { - const tmdbMovie = await this.tmdb.getMediaByImdbId({ + mediaIds.tmdbId = await this.tmdb.resolveImdbIdForScan({ imdbId: result.imdbId, }); - mediaIds.tmdbId = tmdbMovie.id; mediaIds.imdbId = result.imdbId; } } @@ -586,10 +568,10 @@ class PlexScanner if (special.tmdbId) { await this.processPlexMovieByTmdbId(episode, special.tmdbId); } else if (special.imdbId) { - const tmdbMovie = await this.tmdb.getMediaByImdbId({ + const tmdbId = await this.tmdb.resolveImdbIdForScan({ imdbId: special.imdbId, }); - await this.processPlexMovieByTmdbId(episode, tmdbMovie.id); + await this.processPlexMovieByTmdbId(episode, tmdbId); } } } diff --git a/server/lib/scanners/sonarr/index.ts b/server/lib/scanners/sonarr/index.ts index 0702f4dc59..80c1157f7d 100644 --- a/server/lib/scanners/sonarr/index.ts +++ b/server/lib/scanners/sonarr/index.ts @@ -6,6 +6,7 @@ import { ANIME_KEYWORD_ID } from '@server/api/themoviedb/constants'; import type { TmdbKeyword, TmdbTvDetails, + TmdbTvScanDetails, } from '@server/api/themoviedb/interfaces'; import { MediaStatus, MediaType } from '@server/constants/media'; import { getRepository } from '@server/datasource'; @@ -157,18 +158,18 @@ class SonarrScanner try { const mediaRepository = getRepository(Media); const processableSeasons: ProcessableSeason[] = []; - let tvShow: TmdbTvDetails; + let tvShow: TmdbTvScanDetails | TmdbTvDetails; const media = await mediaRepository.findOne({ where: { tvdbId: sonarrSeries.tvdbId }, }); if (!media || !media.tmdbId) { - tvShow = await this.tmdb.getShowByTvdbId({ + tvShow = await this.tmdb.getShowByTvdbIdForScan({ tvdbId: sonarrSeries.tvdbId, }); } else { - tvShow = await this.tmdb.getTvShow({ tvId: media.tmdbId }); + tvShow = await this.tmdb.getTvShowForScan({ tvId: media.tmdbId }); } const tmdbId = tvShow.id; diff --git a/server/lib/scanners/sonarr/sonarr.test.ts b/server/lib/scanners/sonarr/sonarr.test.ts index e012c979eb..eb061e0631 100644 --- a/server/lib/scanners/sonarr/sonarr.test.ts +++ b/server/lib/scanners/sonarr/sonarr.test.ts @@ -89,6 +89,10 @@ TheMovieDb.prototype.getShowByTvdbId = async function (args) { return getShowByTvdbIdImpl(args); }; +TheMovieDb.prototype.getShowByTvdbIdForScan = async function (args) { + return getShowByTvdbIdImpl(args); +}; + let getTvShowImpl: (args: { tvId: number; language?: string; @@ -103,6 +107,15 @@ Object.defineProperty(TheMovieDb.prototype, 'getTvShow', { configurable: true, }); +Object.defineProperty(TheMovieDb.prototype, 'getTvShowForScan', { + set() {}, + get() { + return async (args: { tvId: number; language?: string }) => + getTvShowImpl(args); + }, + configurable: true, +}); + mock.method(MediaRequest, 'sendNotification', async () => undefined); setupTestDb(); diff --git a/src/components/Settings/SettingsJobsCache/index.tsx b/src/components/Settings/SettingsJobsCache/index.tsx index ee8a419889..70db13a5ef 100644 --- a/src/components/Settings/SettingsJobsCache/index.tsx +++ b/src/components/Settings/SettingsJobsCache/index.tsx @@ -578,33 +578,22 @@ const SettingsJobs = () => { - {cacheData?.apiCaches - ?.filter( - (cache) => - !( - settings.currentSettings.mediaServerType !== - MediaServerType.PLEX && cache.id === 'plexguid' - ) - ) - .map((cache) => ( - - {cache.name} - {intl.formatNumber(cache.stats.hits)} - {intl.formatNumber(cache.stats.misses)} - {intl.formatNumber(cache.stats.keys)} - {formatBytes(cache.stats.ksize)} - {formatBytes(cache.stats.vsize)} - - - - - ))} + {cacheData?.apiCaches?.map((cache) => ( + + {cache.name} + {intl.formatNumber(cache.stats.hits)} + {intl.formatNumber(cache.stats.misses)} + {intl.formatNumber(cache.stats.keys)} + {formatBytes(cache.stats.ksize)} + {formatBytes(cache.stats.vsize)} + + + + + ))}