Comicvine Cache

README.md

@@ -175,6 +175,8 @@ metadataProviders:
   comicVineApiKey: # required for comicVine provider https://comicvine.gamespot.com/api/ env:KOMF_METADATA_PROVIDERS_COMIC_VINE_API_KEY
   comicVineSearchLimit: # define ComicVine search result Limit, default is 10
   comicVineIssueName: # string that contains "{number}" which will be replaced by the issue number ie. "Issue #{number}". Used when an issue has no name on ComicVine, default is null
+  cacheDatabaseFile: # cache database file location. default is "./cv_cache.db"
+  cacheDatabaseExpiry: # number of days after which an entry in the cache is considered expired. set it to 0 for unlimited. default is 14
   comicVineIdFormat: # string that contains "{id}" which will serve to parse the ComicVine volume of a given book from its title or folder name ie. "[cv-{id}]" which will correctly identify '.../Uncanny X-Men Omnibus (2006) [cv-27512]' as being [4050-27512](https://comicvine.gamespot.com/uncanny-x-men-omnibus/4050-27512/)
   bangumiToken: # bangumi provider require a token to show nsfw items https://next.bgm.tv/demo/access-token  env:KOMF_METADATA_PROVIDERS_BANGUMI_TOKEN
   defaultProviders:

komf-app/src/main/kotlin/snd/komf/app/api/MetadataRoutes.kt

@@ -49,6 +49,9 @@ class MetadataRoutes(
 
             resetSeriesRoute()
             resetLibraryRoute()
+
+            clearSeriesCacheRoute()
+            clearSeriesIssuesCacheRoute()
         }
     }
 
@@ -144,6 +147,62 @@ class MetadataRoutes(
         }
     }
 
+    private fun Route.clearSeriesCacheRoute() {
+        post("/cache/library/{libraryId}/series/{seriesId}/clear") {
+            val libraryId = call.parameters.getOrFail("libraryId")
+            val seriesId = MediaServerSeriesId(call.parameters.getOrFail("seriesId"))
+            val series = mediaServerClient
+                .first()
+                .getSeries(MediaServerSeriesId(seriesId.value))
+
+            series.metadata.links.forEach {
+                if (it.url.contains("comicvine.gamespot.com")) {
+                    val providerSeriesId = it.url.trimEnd('/').substringAfterLast('-')
+                    metadataServiceProvider
+                        .first()
+                        .metadataServiceFor(libraryId)
+                        .clearSeriesCache(
+                            libraryId,
+                            CoreProviders.COMIC_VINE,
+                            ProviderSeriesId(providerSeriesId),
+                        )
+
+                    call.respond(HttpStatusCode.Accepted, "")
+                }
+            }
+
+            call.respond(HttpStatusCode.NoContent, "")
+        }
+    }
+
+    private fun Route.clearSeriesIssuesCacheRoute() {
+        post("/cache/library/{libraryId}/series/{seriesId}/clearissues") {
+            val libraryId = call.parameters.getOrFail("libraryId")
+            val seriesId = MediaServerSeriesId(call.parameters.getOrFail("seriesId"))
+            val series = mediaServerClient
+                .first()
+                .getSeries(MediaServerSeriesId(seriesId.value))
+
+            series.metadata.links.forEach {
+                if (it.url.contains("comicvine.gamespot.com")) {
+                    val providerSeriesId = it.url.trimEnd('/').substringAfterLast('-')
+                    metadataServiceProvider
+                        .first()
+                        .metadataServiceFor(libraryId)
+                        .clearSeriesIssuesCache(
+                            libraryId,
+                            CoreProviders.COMIC_VINE,
+                            ProviderSeriesId(providerSeriesId),
+                        )
+
+                    call.respond(HttpStatusCode.Accepted, "")
+                }
+            }
+
+            call.respond(HttpStatusCode.NoContent, "")
+        }
+    }
+
     private fun Route.matchSeriesRoute() {
         post("/match/library/{libraryId}/series/{seriesId}") {
 
@@ -190,4 +249,4 @@ class MetadataRoutes(
         }
     }
 
-}
+}

komf-app/src/main/kotlin/snd/komf/app/api/deprecated/DeprecatedMetadataRoutes.kt

@@ -41,6 +41,8 @@ class DeprecatedMetadataRoutes(
                 matchLibraryRoute()
                 resetSeriesRoute()
                 resetLibraryRoute()
+                clearSeriesCacheRoute()
+                clearSeriesIssuesCacheRoute()
             }
         }
     }
@@ -115,6 +117,62 @@ class DeprecatedMetadataRoutes(
         }
     }
 
+    private fun Route.clearSeriesCacheRoute() {
+        post("/cache/library/{libraryId}/series/{seriesId}/clear") {
+            val libraryId = call.parameters.getOrFail("libraryId")
+            val seriesId = MediaServerSeriesId(call.parameters.getOrFail("seriesId"))
+            val series = mediaServerClient
+                .first()
+                .getSeries(MediaServerSeriesId(seriesId.value))
+
+            series.metadata.links.forEach {
+                if (it.url.contains("comicvine.gamespot.com")) {
+                    val providerSeriesId = it.url.trimEnd('/').substringAfterLast('-')
+                    metadataServiceProvider
+                        .first()
+                        .metadataServiceFor(libraryId)
+                        .clearSeriesCache(
+                            libraryId,
+                            CoreProviders.COMIC_VINE,
+                            ProviderSeriesId(providerSeriesId),
+                        )
+
+                    call.respond(HttpStatusCode.Accepted, "")
+                }
+            }
+
+            call.respond(HttpStatusCode.NoContent, "")
+        }
+    }
+
+    private fun Route.clearSeriesIssuesCacheRoute() {
+        post("/cache/library/{libraryId}/series/{seriesId}/clearissues") {
+            val libraryId = call.parameters.getOrFail("libraryId")
+            val seriesId = MediaServerSeriesId(call.parameters.getOrFail("seriesId"))
+            val series = mediaServerClient
+                .first()
+                .getSeries(MediaServerSeriesId(seriesId.value))
+
+            series.metadata.links.forEach {
+                if (it.url.contains("comicvine.gamespot.com")) {
+                    val providerSeriesId = it.url.trimEnd('/').substringAfterLast('-')
+                    metadataServiceProvider
+                        .first()
+                        .metadataServiceFor(libraryId)
+                        .clearSeriesIssuesCache(
+                            libraryId,
+                            CoreProviders.COMIC_VINE,
+                            ProviderSeriesId(providerSeriesId),
+                        )
+
+                    call.respond(HttpStatusCode.Accepted, "")
+                }
+            }
+
+            call.respond(HttpStatusCode.NoContent, "")
+        }
+    }
+
     private fun Route.matchLibraryRoute() {
         post("/match/library/{libraryId}") {
             val libraryId = MediaServerLibraryId(call.parameters.getOrFail("libraryId"))

komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvider.kt

@@ -15,6 +15,10 @@ interface MetadataProvider {
 
     suspend fun getSeriesCover(seriesId: ProviderSeriesId): Image?
 
+    suspend fun clearSeriesCache(providerSeriesId: ProviderSeriesId)
+
+    suspend fun clearSeriesIssuesCache(providerSeriesId: ProviderSeriesId)
+
     suspend fun getBookMetadata(seriesId: ProviderSeriesId, bookId: ProviderBookId): ProviderBookMetadata
 
     suspend fun searchSeries(seriesName: String, limit: Int = 5): Collection<SeriesSearchResult>

komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt

@@ -25,6 +25,8 @@ data class MetadataProvidersConfig(
     val defaultProviders: ProvidersConfig = ProvidersConfig(),
     val libraryProviders: Map<String, ProvidersConfig> = emptyMap(),
     val mangabakaDatabaseDir: String = "./mangabaka",
+    val cacheDatabaseFile: String = "./cv_cache.db",
+    val cacheDatabaseExpiry: Int = 14,
 )
 
 @Serializable

komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt

@@ -108,6 +108,8 @@ class ProvidersModule(
             comicVineIssueName = config.comicVineIssueName,
             comicVineIdFormat = config.comicVineIdFormat,
             bangumiToken = config.bangumiToken,
+            cacheDatabaseFile = config.cacheDatabaseFile,
+            cacheDatabaseExpiry = config.cacheDatabaseExpiry,
         )
         val libraryProviders = config.libraryProviders
             .map { (libraryId, libraryConfig) ->
@@ -120,6 +122,8 @@ class ProvidersModule(
                     comicVineIssueName = config.comicVineIssueName,
                     comicVineIdFormat = config.comicVineIdFormat,
                     bangumiToken = config.bangumiToken,
+                    cacheDatabaseFile = config.cacheDatabaseFile,
+                    cacheDatabaseExpiry = config.cacheDatabaseExpiry,
                 )
             }
             .toMap()
@@ -333,6 +337,8 @@ class ProvidersModule(
         comicVineIssueName: String?,
         comicVineIdFormat: String?,
         bangumiToken: String?,
+        cacheDatabaseFile: String,
+        cacheDatabaseExpiry: Int,
     ): MetadataProvidersContainer {
         return MetadataProvidersContainer(
             mangaupdates = createMangaUpdatesMetadataProvider(
@@ -403,6 +409,8 @@ class ProvidersModule(
                 comicVineIdFormat = comicVineIdFormat,
                 rateLimiter = comicVineRateLimiter,
                 defaultNameMatcher = defaultNameMatcher,
+                cacheDatabaseFile = cacheDatabaseFile,
+                cacheDatabaseExpiry = cacheDatabaseExpiry,
             ),
             comicVinePriority = config.comicVine.priority,
             hentag = createHentagMetadataProvider(
@@ -706,6 +714,8 @@ class ProvidersModule(
         comicVineIdFormat: String?,
         rateLimiter: ComicVineRateLimiter,
         defaultNameMatcher: NameSimilarityMatcher,
+        cacheDatabaseFile: String,
+        cacheDatabaseExpiry: Int,
     ): ComicVineMetadataProvider? {
         if (config.enabled.not()) return null
         requireNotNull(apiKey) { "Api key is not configured for ComicVine provider" }
@@ -719,7 +729,9 @@ class ProvidersModule(
             },
             apiKey = apiKey,
             comicVineSearchLimit = comicVineSearchLimit,
-            rateLimiter = rateLimiter
+            rateLimiter = rateLimiter,
+            cacheDatabaseFile = cacheDatabaseFile,
+            cacheDatabaseExpiry = cacheDatabaseExpiry,
         )
         val metadataMapper = ComicVineMetadataMapper(
             seriesMetadataConfig = config.seriesMetadata,
@@ -908,4 +920,4 @@ class ProvidersModule(
     }
 
 
-}
+}

komf-core/src/commonMain/kotlin/snd/komf/providers/anilist/AniListMetadataProvider.kt

@@ -42,6 +42,14 @@ class AniListMetadataProvider(
         return client.getThumbnail(series)
     }
 
+    override suspend fun clearSeriesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
+    override suspend fun clearSeriesIssuesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
     override suspend fun getBookMetadata(seriesId: ProviderSeriesId, bookId: ProviderBookId): ProviderBookMetadata {
         throw UnsupportedOperationException()
     }

komf-core/src/commonMain/kotlin/snd/komf/providers/bangumi/BangumiMetadataProvider.kt

@@ -46,6 +46,14 @@ class BangumiMetadataProvider(
         return client.getThumbnail(series)
     }
 
+    override suspend fun clearSeriesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
+    override suspend fun clearSeriesIssuesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
     override suspend fun getBookMetadata(seriesId: ProviderSeriesId, bookId: ProviderBookId): ProviderBookMetadata {
         val book = client.getSubject(bookId.id.toLong())
         val thumbnail = if (fetchSeriesCovers) client.getThumbnail(book) else null

komf-core/src/commonMain/kotlin/snd/komf/providers/bookwalker/BookWalkerMetadataProvider.kt

@@ -66,6 +66,14 @@ class BookWalkerMetadataProvider(
         return fetchCover(getFirstBook(books))
     }
 
+    override suspend fun clearSeriesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
+    override suspend fun clearSeriesIssuesCache(providerSeriesId: ProviderSeriesId) {
+        throw UnsupportedOperationException()
+    }
+
     override suspend fun getBookMetadata(seriesId: ProviderSeriesId, bookId: ProviderBookId): ProviderBookMetadata {
         val bookMetadata = bookCache.get(BookWalkerBookId(bookId.id)) { client.getBook(BookWalkerBookId(bookId.id)) }
         val bookCover = if (fetchBookCovers) fetchCover(bookMetadata) else null

komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineCache.kt

@@ -0,0 +1,108 @@
+package snd.komf.providers.comicvine
+
+import org.jetbrains.exposed.v1.jdbc.Database
+import org.jetbrains.exposed.v1.jdbc.SchemaUtils
+import org.jetbrains.exposed.v1.core.Table
+import org.jetbrains.exposed.v1.core.and
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.core.greater
+import org.jetbrains.exposed.v1.datetime.*
+import org.jetbrains.exposed.v1.jdbc.deleteWhere
+import org.jetbrains.exposed.v1.jdbc.select
+import org.jetbrains.exposed.v1.jdbc.upsert
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import java.nio.file.Path
+import java.io.File
+import java.time.temporal.ChronoUnit
+import kotlin.time.Clock
+import kotlin.time.Duration
+import kotlin.time.Instant
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.toInstant
+import kotlinx.datetime.toLocalDateTime
+import kotlinx.datetime.plus
+import kotlinx.datetime.DateTimeUnit
+
+object QueriesTable : Table("queries") {
+    val urlCol = text("url")
+    override val primaryKey = PrimaryKey(urlCol)
+
+    val createdAtCol = timestamp("created_at")
+
+    val responseCol = text("response")
+}
+
+class ComicVineCache(
+    private val databaseFile: String,
+    private val expiry: Int,
+) {
+    private val databasePath = Path.of(databaseFile)
+    private val database = Database.connect("jdbc:sqlite:$databasePath", driver = "org.sqlite.JDBC")
+
+    init {
+        transaction(db = database) {
+            SchemaUtils.create(QueriesTable)
+        }
+    }
+
+    private fun getExpiryTimestamp(): Instant {
+        return Clock.System.now()
+            .toLocalDateTime(TimeZone.UTC)
+            .toInstant(TimeZone.UTC)
+            .plus(value = expiry * 24, DateTimeUnit.HOUR)
+    }
+
+    private fun getNowTimestamp(): Instant {
+        return Clock.System.now()
+            .toLocalDateTime(TimeZone.UTC)
+            .toInstant(TimeZone.UTC)
+    }
+
+    private fun maskApiKey(url: String): String {
+        return url.replace(
+            Regex("""api_key=[^&]+"""),
+            "api_key=*****"
+        )
+    }
+
+    fun addEntry(url: String, response: String) {
+        transaction(db = database) {
+            QueriesTable.upsert {
+                it[urlCol] = maskApiKey(url)
+                it[responseCol] = response
+                it[createdAtCol] = getExpiryTimestamp()
+            }
+        }
+    }
+
+    fun removeEntry(url: String) {
+        transaction(db = database) {
+            QueriesTable.deleteWhere {
+                QueriesTable.urlCol eq maskApiKey(url)
+            }
+        }
+    }
+
+    suspend fun getEntry(url: String): String? {
+        if (expiry == 0) {
+            return transaction(db = database) {
+                QueriesTable
+                    .select(QueriesTable.responseCol).where {
+                        QueriesTable.urlCol eq maskApiKey(url)
+                    }
+                    .firstOrNull()
+                    ?.get(QueriesTable.responseCol)
+            }
+        }
+
+        return transaction(db = database) {
+            QueriesTable
+                .select(QueriesTable.responseCol).where {
+                    (QueriesTable.urlCol eq maskApiKey(url)) and
+                    (QueriesTable.createdAtCol greater getNowTimestamp())
+                }
+                .firstOrNull()
+                ?.get(QueriesTable.responseCol)
+        }
+    }
+}