Fixes, improvements & Fix for LiveChart Website changes

analyzer/build.gradle.kts

@@ -51,6 +51,8 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach
 
 tasks.withType<Test> {
     useJUnitPlatform()
+    // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag.
+    jvmArgs("--enable-native-access=ALL-UNNAMED")
     reports.html.required.set(false)
     reports.junitXml.required.set(true)
     maxParallelForks = rootProject.extra["maxParallelForks"] as Int

analyzer/src/main/resources/logback.xml

@@ -1,5 +1,5 @@
 <configuration>
-    <property name="MSG_PATTERN" value="%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n" />
+    <property name="MSG_PATTERN" value="%d{HH:mm:ss.SSS} [%thread] %-5level %replace(%logger{0}){'\$Companion', ''} - %msg%n" />
     <timestamp key="timestamp" datePattern="yyyy-MM-dd_-_HHmmss"/>
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">

anidb/build.gradle.kts

@@ -52,19 +52,21 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach
 
 tasks.withType<Test> {
     useJUnitPlatform()
+    // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag.
+    jvmArgs("--enable-native-access=ALL-UNNAMED")
     reports.html.required.set(false)
     reports.junitXml.required.set(true)
     maxParallelForks = rootProject.extra["maxParallelForks"] as Int
     systemProperty("junit.jupiter.execution.parallel.enabled", "true")
     systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
 }
 
-val sourcesJar by tasks.registering(Jar::class) {
+val sourcesJar = tasks.register<Jar>("sourcesJar") {
     archiveClassifier.set("sources")
     from(sourceSets.main.get().allSource)
 }
 
-val javaDoc by tasks.registering(Jar::class) {
+val javaDoc = tasks.register<Jar>("javaDoc") {
     archiveClassifier.set("javadoc")
     from(sourceSets.main.get().allSource)
 }

anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt

@@ -4,13 +4,19 @@ import io.github.manamiproject.modb.core.config.AnimeId
 import io.github.manamiproject.modb.core.config.ConfigRegistry
 import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
 import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
+import io.github.manamiproject.modb.core.config.IntPropertyDelegate
 import io.github.manamiproject.modb.core.config.StringPropertyDelegate
 import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK
 import io.github.manamiproject.modb.core.downloader.Downloader
 import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
 import io.github.manamiproject.modb.core.httpclient.HttpClient
 import io.github.manamiproject.modb.core.logging.LoggerDelegate
+import io.github.manamiproject.modb.core.excludeFromTestContext
+import io.github.manamiproject.modb.core.random
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.withContext
+import kotlin.time.DurationUnit.MILLISECONDS
+import kotlin.time.toDuration
 import java.io.ByteArrayInputStream
 import java.net.URI
 import java.util.zip.GZIPInputStream
@@ -23,7 +29,7 @@ import java.util.zip.GZIPInputStream
  * configuration rather than constants.
  * @since 1.0.0
  * @property metaDataProviderConfig Configuration for downloading data.
- * @property configRegistry Source of the registered client name and version.
+ * @property configRegistry Source of the registered client name and version and of `modb.anidb.banRetries`, the number of retries while banned. **Default:** `5`
  * @property httpClient To actually download the anime data.
  */
 public class AnidbApiDownloader(
@@ -42,9 +48,44 @@ public class AnidbApiDownloader(
         namespace = CONFIG_NAMESPACE,
     )
 
+    private val banRetries: Int by IntPropertyDelegate(
+        configRegistry = configRegistry,
+        namespace = CONFIG_NAMESPACE,
+        default = DEFAULT_BAN_RETRIES,
+        validator = { it >= 0 },
+    )
+
     override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = withContext(LIMITED_NETWORK) {
         log.debug { "Downloading [anidbId=$id]" }
 
+        var attempt = 0
+
+        while (true) {
+            try {
+                return@withContext downloadOnce(id, onDeadEntry)
+            } catch (e: AnidbBannedException) {
+                attempt++
+
+                if (attempt > banRetries) {
+                    log.error { "Still banned by the API after [$banRetries] retries. Giving up on [anidbId=$id]." }
+                    throw e
+                }
+
+                // Same waiting time as the retries of the http client. The ban is answered with
+                // http 200, so the http client itself doesn't see it as a reason to retry.
+                val waitTime = (random(MIN_BAN_WAIT_IN_MILLISECONDS, MAX_BAN_WAIT_IN_MILLISECONDS) * attempt).toDuration(MILLISECONDS)
+                log.warn { "Banned by the API. Retry [$attempt/$banRetries] for [anidbId=$id] in [$waitTime]." }
+
+                excludeFromTestContext(metaDataProviderConfig) {
+                    delay(waitTime)
+                }
+            }
+        }
+
+        EMPTY_RESPONSE
+    }
+
+    private suspend fun downloadOnce(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String {
         val url = URI("${metaDataProviderConfig.buildDataDownloadLink(id)}&client=$client&clientver=$clientver&protover=$PROTOCOL_VERSION").toURL()
         val response = httpClient.get(url)
         val body = decompressIfNecessary(response.bodyAsByteArray())
@@ -62,15 +103,20 @@ public class AnidbApiDownloader(
             // <error>Anime not found</error>, so a check on the code alone lets that through and the
             // apology page ends up stored and converted as if it were an anime.
             if (code in DEAD_ENTRY_CODES || message.lowercase() in DEAD_ENTRY_MESSAGES) {
-                log.info { "Adding [anidbId=$id] to dead entries list, because the API responded with [$message]." }
+                // Whether the id is dead or still pending is decided by the caller, which logs the outcome.
+                log.info { "The API responded with [$message] for [anidbId=$id]." }
                 onDeadEntry.invoke(id)
-                return@withContext EMPTY_RESPONSE
+                return EMPTY_RESPONSE
+            }
+
+            if (message.lowercase() == BANNED_MESSAGE) {
+                throw AnidbBannedException("API responded with error [code=$code], [message=$message] for [anidbId=$id].")
             }
 
             throw IllegalStateException("API responded with error [code=$code], [message=$message] for [anidbId=$id].")
         }
 
-        return@withContext body
+        return body
     }
 
     private fun decompressIfNecessary(body: ByteArray): String {
@@ -88,6 +134,10 @@ public class AnidbApiDownloader(
         private val log by LoggerDelegate()
 
         private const val PROTOCOL_VERSION = 1
+        private const val BANNED_MESSAGE = "banned"
+        private const val DEFAULT_BAN_RETRIES = 5
+        private const val MIN_BAN_WAIT_IN_MILLISECONDS = 60000
+        private const val MAX_BAN_WAIT_IN_MILLISECONDS = 90000
         private const val EMPTY_RESPONSE = ""
         private const val GZIP_MAGIC_FIRST: Byte = 0x1f
         private const val GZIP_MAGIC_SECOND: Byte = 0x8b.toByte()

anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt

@@ -0,0 +1,9 @@
+package io.github.manamiproject.modb.anidb
+
+/**
+ * Thrown if the AniDB API answers with `<error code="500">banned</error>`.
+ * The API sends the ban with http status 200 and without any information on how long it lasts.
+ * @since 20.0.0
+ * @param message Description including the anime ID which was requested.
+ */
+public class AnidbBannedException(message: String) : RuntimeException(message)

anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt

@@ -14,6 +14,7 @@ import kotlinx.coroutines.withContext
 /**
  * Downloads anime data from anidb.net
  * In case the anime addition is pending, the [download] function will return a static [String] (see [ANIDB_PENDING_FILE_INDICATOR]).
+ * In case the anime is 18+ restricted, it returns [ANIDB_RESTRICTED_INDICATOR] instead.
  * The caller must handle this behavior.
  * @since 1.0.0
  * @property metaDataProviderConfig Configuration for downloading data.
@@ -39,7 +40,12 @@ public class AnidbDownloader(
         check(response.isOk() || response.code == 404) { "Unexpected response code [anidbId=$id], [responseCode=${response.code}]" }
 
         return@withContext when {
-            responseChecker.isHentai() || responseChecker.isRemovedFromAnidb() -> {
+            // 18+ pages show no data without an account: skipped, neither dead nor pending
+            responseChecker.isHentai() -> {
+                log.info { "Skipping [anidbId=$id], because it is 18+ restricted." }
+                ANIDB_RESTRICTED_INDICATOR
+            }
+            responseChecker.isRemovedFromAnidb() -> {
                 log.info { "Adding [anidbId=$id] to dead-entries list" }
                 onDeadEntry.invoke(id)
                 EMPTY
@@ -59,9 +65,15 @@ public class AnidbDownloader(
         public val instance: AnidbDownloader by lazy { AnidbDownloader() }
 
         /**
-         * Indicator for pending files which have been downloaded, but cannot be converted.
+         * Returned for an entry whose addition is still pending. There is nothing to convert.
          * @since 7.0.0
          */
         public const val ANIDB_PENDING_FILE_INDICATOR: String = ">pending<"
+
+        /**
+         * Returned for an entry which is 18+ restricted. The page shows no data without an account.
+         * @since 7.0.0
+         */
+        public const val ANIDB_RESTRICTED_INDICATOR: String = ">restricted<"
     }
 }

anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt

@@ -0,0 +1,144 @@
+package io.github.manamiproject.modb.anidb
+
+import io.github.manamiproject.modb.core.config.ConfigRegistry
+import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
+import io.github.manamiproject.modb.core.httpclient.HttpClient
+import io.github.manamiproject.modb.core.httpclient.HttpResponse
+import io.github.manamiproject.modb.core.httpclient.RequestBody
+import io.github.manamiproject.modb.core.httpclient.RetryCase
+import io.github.manamiproject.modb.test.exceptionExpected
+import io.github.manamiproject.modb.test.shouldNotBeInvoked
+import kotlinx.coroutines.test.runTest
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Nested
+import java.net.URL
+import kotlin.test.Test
+
+internal class AnidbApiDownloaderTest {
+
+    @Nested
+    inner class BannedTests {
+
+        @Test
+        fun `retries if the API responds with banned and returns the body once the ban is over`() {
+            runTest {
+                // given
+                var invocations = 0
+                val testHttpClient = respondingWith {
+                    invocations++
+                    when (invocations) {
+                        1, 2 -> """<error code="500">banned</error>"""
+                        else -> """<anime id="1"></anime>"""
+                    }
+                }
+
+                val downloader = AnidbApiDownloader(
+                    metaDataProviderConfig = testAnidbConfig,
+                    configRegistry = testConfigRegistry,
+                    httpClient = testHttpClient,
+                )
+
+                // when
+                val result = downloader.download("1") { shouldNotBeInvoked() }
+
+                // then
+                assertThat(result).isEqualTo("""<anime id="1"></anime>""")
+                assertThat(invocations).isEqualTo(3)
+            }
+        }
+
+        @Test
+        fun `throws AnidbBannedException if the ban is still active after all retries`() {
+            // given
+            var invocations = 0
+            val testHttpClient = respondingWith {
+                invocations++
+                """<error code="500">banned</error>"""
+            }
+
+            val downloader = AnidbApiDownloader(
+                metaDataProviderConfig = testAnidbConfig,
+                configRegistry = testConfigRegistry,
+                httpClient = testHttpClient,
+            )
+
+            // when
+            val result = exceptionExpected<AnidbBannedException> {
+                downloader.download("1") { shouldNotBeInvoked() }
+            }
+
+            // then
+            assertThat(result).hasMessage("API responded with error [code=500], [message=banned] for [anidbId=1].")
+            assertThat(invocations).isEqualTo(6)
+        }
+
+        @Test
+        fun `number of retries while banned can be set via banRetries`() {
+            // given
+            var invocations = 0
+            val testHttpClient = respondingWith {
+                invocations++
+                """<error code="500">banned</error>"""
+            }
+
+            val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry {
+                override fun string(key: String): String = "test"
+                override fun int(key: String): Int? = if (key == "modb.anidb.banRetries") 2 else null
+            }
+
+            val downloader = AnidbApiDownloader(
+                metaDataProviderConfig = testAnidbConfig,
+                configRegistry = testConfigRegistry,
+                httpClient = testHttpClient,
+            )
+
+            // when
+            exceptionExpected<AnidbBannedException> {
+                downloader.download("1") { shouldNotBeInvoked() }
+            }
+
+            // then
+            assertThat(invocations).isEqualTo(3)
+        }
+
+        @Test
+        fun `other errors are not retried`() {
+            // given
+            var invocations = 0
+            val testHttpClient = respondingWith {
+                invocations++
+                """<error code="302">client version missing or invalid</error>"""
+            }
+
+            val downloader = AnidbApiDownloader(
+                metaDataProviderConfig = testAnidbConfig,
+                configRegistry = testConfigRegistry,
+                httpClient = testHttpClient,
+            )
+
+            // when
+            val result = exceptionExpected<IllegalStateException> {
+                downloader.download("1") { shouldNotBeInvoked() }
+            }
+
+            // then
+            assertThat(result).hasMessage("API responded with error [code=302], [message=client version missing or invalid] for [anidbId=1].")
+            assertThat(invocations).isEqualTo(1)
+        }
+    }
+
+    private val testAnidbConfig = object: MetaDataProviderConfig by AnidbConfig {
+        override fun isTestContext(): Boolean = true
+    }
+
+    private val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry {
+        override fun string(key: String): String = "test"
+        override fun int(key: String): Int? = null
+    }
+
+    private fun respondingWith(body: () -> String) = object: HttpClient {
+        override suspend fun post(url: URL, requestBody: RequestBody, headers: Map<String, Collection<String>>): HttpResponse = shouldNotBeInvoked()
+        override suspend fun get(url: URL, headers: Map<String, Collection<String>>): HttpResponse = HttpResponse(200, body())
+        override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = shouldNotBeInvoked()
+    }
+}

anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt

@@ -3,6 +3,7 @@ package io.github.manamiproject.modb.anidb
 import com.github.tomakehurst.wiremock.WireMockServer
 import com.github.tomakehurst.wiremock.client.WireMock.*
 import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR
+import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR
 import io.github.manamiproject.modb.core.config.AnimeId
 import io.github.manamiproject.modb.core.config.FileSuffix
 import io.github.manamiproject.modb.core.config.Hostname
@@ -156,7 +157,7 @@ internal class AnidbDownloaderTest : MockServerTestCase<WireMockServer> by WireM
         }
 
         @Test
-        fun `invoke onDeadEntry if the response body indicates a hentai`() {
+        fun `returns the restricted indicator for an 18+ entry without invoking onDeadEntry`() {
             runTest {
                 // given
                 val id = "11376"
@@ -188,8 +189,8 @@ internal class AnidbDownloaderTest : MockServerTestCase<WireMockServer> by WireM
                 }
 
                 // then
-                assertThat(result).isEmpty()
-                assertThat(deadEntry).isEqualTo(id)
+                assertThat(result).isEqualTo(ANIDB_RESTRICTED_INDICATOR)
+                assertThat(deadEntry).isEmpty()
             }
         }
 

anilist/build.gradle.kts

@@ -42,19 +42,21 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach
 
 tasks.withType<Test> {
     useJUnitPlatform()
+    // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag.
+    jvmArgs("--enable-native-access=ALL-UNNAMED")
     reports.html.required.set(false)
     reports.junitXml.required.set(true)
     maxParallelForks = rootProject.extra["maxParallelForks"] as Int
     systemProperty("junit.jupiter.execution.parallel.enabled", "true")
     systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
 }
 
-val sourcesJar by tasks.registering(Jar::class) {
+val sourcesJar = tasks.register<Jar>("sourcesJar") {
     archiveClassifier.set("sources")
     from(sourceSets.main.get().allSource)
 }
 
-val javaDoc by tasks.registering(Jar::class) {
+val javaDoc = tasks.register<Jar>("javaDoc") {
     archiveClassifier.set("javadoc")
     from(sourceSets.main.get().allSource)
 }