diff --git a/analyzer/build.gradle.kts b/analyzer/build.gradle.kts index 02c83e57..acf6e2a4 100644 --- a/analyzer/build.gradle.kts +++ b/analyzer/build.gradle.kts @@ -51,6 +51,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/analyzer/src/main/resources/logback.xml b/analyzer/src/main/resources/logback.xml index e56c26eb..269bba9e 100644 --- a/analyzer/src/main/resources/logback.xml +++ b/analyzer/src/main/resources/logback.xml @@ -1,5 +1,5 @@ - + diff --git a/anidb/build.gradle.kts b/anidb/build.gradle.kts index 7e854401..880b7cfb 100644 --- a/anidb/build.gradle.kts +++ b/anidb/build.gradle.kts @@ -52,6 +52,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -59,12 +61,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt index faa972b3..ada20846 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt @@ -4,13 +4,19 @@ import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.DefaultConfigRegistry import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.config.StringPropertyDelegate import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK import io.github.manamiproject.modb.core.downloader.Downloader import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.logging.LoggerDelegate +import io.github.manamiproject.modb.core.excludeFromTestContext +import io.github.manamiproject.modb.core.random +import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import kotlin.time.DurationUnit.MILLISECONDS +import kotlin.time.toDuration import java.io.ByteArrayInputStream import java.net.URI import java.util.zip.GZIPInputStream @@ -23,7 +29,7 @@ import java.util.zip.GZIPInputStream * configuration rather than constants. * @since 1.0.0 * @property metaDataProviderConfig Configuration for downloading data. - * @property configRegistry Source of the registered client name and version. + * @property configRegistry Source of the registered client name and version and of `modb.anidb.banRetries`, the number of retries while banned. **Default:** `5` * @property httpClient To actually download the anime data. */ public class AnidbApiDownloader( @@ -42,9 +48,44 @@ public class AnidbApiDownloader( namespace = CONFIG_NAMESPACE, ) + private val banRetries: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = CONFIG_NAMESPACE, + default = DEFAULT_BAN_RETRIES, + validator = { it >= 0 }, + ) + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = withContext(LIMITED_NETWORK) { log.debug { "Downloading [anidbId=$id]" } + var attempt = 0 + + while (true) { + try { + return@withContext downloadOnce(id, onDeadEntry) + } catch (e: AnidbBannedException) { + attempt++ + + if (attempt > banRetries) { + log.error { "Still banned by the API after [$banRetries] retries. Giving up on [anidbId=$id]." } + throw e + } + + // Same waiting time as the retries of the http client. The ban is answered with + // http 200, so the http client itself doesn't see it as a reason to retry. + val waitTime = (random(MIN_BAN_WAIT_IN_MILLISECONDS, MAX_BAN_WAIT_IN_MILLISECONDS) * attempt).toDuration(MILLISECONDS) + log.warn { "Banned by the API. Retry [$attempt/$banRetries] for [anidbId=$id] in [$waitTime]." } + + excludeFromTestContext(metaDataProviderConfig) { + delay(waitTime) + } + } + } + + EMPTY_RESPONSE + } + + private suspend fun downloadOnce(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { val url = URI("${metaDataProviderConfig.buildDataDownloadLink(id)}&client=$client&clientver=$clientver&protover=$PROTOCOL_VERSION").toURL() val response = httpClient.get(url) val body = decompressIfNecessary(response.bodyAsByteArray()) @@ -62,15 +103,20 @@ public class AnidbApiDownloader( // Anime not found, so a check on the code alone lets that through and the // apology page ends up stored and converted as if it were an anime. if (code in DEAD_ENTRY_CODES || message.lowercase() in DEAD_ENTRY_MESSAGES) { - log.info { "Adding [anidbId=$id] to dead entries list, because the API responded with [$message]." } + // Whether the id is dead or still pending is decided by the caller, which logs the outcome. + log.info { "The API responded with [$message] for [anidbId=$id]." } onDeadEntry.invoke(id) - return@withContext EMPTY_RESPONSE + return EMPTY_RESPONSE + } + + if (message.lowercase() == BANNED_MESSAGE) { + throw AnidbBannedException("API responded with error [code=$code], [message=$message] for [anidbId=$id].") } throw IllegalStateException("API responded with error [code=$code], [message=$message] for [anidbId=$id].") } - return@withContext body + return body } private fun decompressIfNecessary(body: ByteArray): String { @@ -88,6 +134,10 @@ public class AnidbApiDownloader( private val log by LoggerDelegate() private const val PROTOCOL_VERSION = 1 + private const val BANNED_MESSAGE = "banned" + private const val DEFAULT_BAN_RETRIES = 5 + private const val MIN_BAN_WAIT_IN_MILLISECONDS = 60000 + private const val MAX_BAN_WAIT_IN_MILLISECONDS = 90000 private const val EMPTY_RESPONSE = "" private const val GZIP_MAGIC_FIRST: Byte = 0x1f private const val GZIP_MAGIC_SECOND: Byte = 0x8b.toByte() diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt new file mode 100644 index 00000000..237eaa55 --- /dev/null +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt @@ -0,0 +1,9 @@ +package io.github.manamiproject.modb.anidb + +/** + * Thrown if the AniDB API answers with `banned`. + * The API sends the ban with http status 200 and without any information on how long it lasts. + * @since 20.0.0 + * @param message Description including the anime ID which was requested. + */ +public class AnidbBannedException(message: String) : RuntimeException(message) diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt index 7ea1bc5f..fcf15e64 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt @@ -14,6 +14,7 @@ import kotlinx.coroutines.withContext /** * Downloads anime data from anidb.net * In case the anime addition is pending, the [download] function will return a static [String] (see [ANIDB_PENDING_FILE_INDICATOR]). + * In case the anime is 18+ restricted, it returns [ANIDB_RESTRICTED_INDICATOR] instead. * The caller must handle this behavior. * @since 1.0.0 * @property metaDataProviderConfig Configuration for downloading data. @@ -39,7 +40,12 @@ public class AnidbDownloader( check(response.isOk() || response.code == 404) { "Unexpected response code [anidbId=$id], [responseCode=${response.code}]" } return@withContext when { - responseChecker.isHentai() || responseChecker.isRemovedFromAnidb() -> { + // 18+ pages show no data without an account: skipped, neither dead nor pending + responseChecker.isHentai() -> { + log.info { "Skipping [anidbId=$id], because it is 18+ restricted." } + ANIDB_RESTRICTED_INDICATOR + } + responseChecker.isRemovedFromAnidb() -> { log.info { "Adding [anidbId=$id] to dead-entries list" } onDeadEntry.invoke(id) EMPTY @@ -59,9 +65,15 @@ public class AnidbDownloader( public val instance: AnidbDownloader by lazy { AnidbDownloader() } /** - * Indicator for pending files which have been downloaded, but cannot be converted. + * Returned for an entry whose addition is still pending. There is nothing to convert. * @since 7.0.0 */ public const val ANIDB_PENDING_FILE_INDICATOR: String = ">pending<" + + /** + * Returned for an entry which is 18+ restricted. The page shows no data without an account. + * @since 7.0.0 + */ + public const val ANIDB_RESTRICTED_INDICATOR: String = ">restricted<" } } \ No newline at end of file diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt new file mode 100644 index 00000000..1068c4de --- /dev/null +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt @@ -0,0 +1,144 @@ +package io.github.manamiproject.modb.anidb + +import io.github.manamiproject.modb.core.config.ConfigRegistry +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.httpclient.HttpClient +import io.github.manamiproject.modb.core.httpclient.HttpResponse +import io.github.manamiproject.modb.core.httpclient.RequestBody +import io.github.manamiproject.modb.core.httpclient.RetryCase +import io.github.manamiproject.modb.test.exceptionExpected +import io.github.manamiproject.modb.test.shouldNotBeInvoked +import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Nested +import java.net.URL +import kotlin.test.Test + +internal class AnidbApiDownloaderTest { + + @Nested + inner class BannedTests { + + @Test + fun `retries if the API responds with banned and returns the body once the ban is over`() { + runTest { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + when (invocations) { + 1, 2 -> """banned""" + else -> """""" + } + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = downloader.download("1") { shouldNotBeInvoked() } + + // then + assertThat(result).isEqualTo("""""") + assertThat(invocations).isEqualTo(3) + } + } + + @Test + fun `throws AnidbBannedException if the ban is still active after all retries`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """banned""" + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(result).hasMessage("API responded with error [code=500], [message=banned] for [anidbId=1].") + assertThat(invocations).isEqualTo(6) + } + + @Test + fun `number of retries while banned can be set via banRetries`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """banned""" + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun string(key: String): String = "test" + override fun int(key: String): Int? = if (key == "modb.anidb.banRetries") 2 else null + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(invocations).isEqualTo(3) + } + + @Test + fun `other errors are not retried`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """client version missing or invalid""" + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(result).hasMessage("API responded with error [code=302], [message=client version missing or invalid] for [anidbId=1].") + assertThat(invocations).isEqualTo(1) + } + } + + private val testAnidbConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + private val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun string(key: String): String = "test" + override fun int(key: String): Int? = null + } + + private fun respondingWith(body: () -> String) = object: HttpClient { + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun get(url: URL, headers: Map>): HttpResponse = HttpResponse(200, body()) + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = shouldNotBeInvoked() + } +} diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt index 40c5e60d..3b130d7a 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt @@ -3,6 +3,7 @@ package io.github.manamiproject.modb.anidb import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.WireMock.* import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.FileSuffix import io.github.manamiproject.modb.core.config.Hostname @@ -156,7 +157,7 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM } @Test - fun `invoke onDeadEntry if the response body indicates a hentai`() { + fun `returns the restricted indicator for an 18+ entry without invoking onDeadEntry`() { runTest { // given val id = "11376" @@ -188,8 +189,8 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM } // then - assertThat(result).isEmpty() - assertThat(deadEntry).isEqualTo(id) + assertThat(result).isEqualTo(ANIDB_RESTRICTED_INDICATOR) + assertThat(deadEntry).isEmpty() } } diff --git a/anilist/build.gradle.kts b/anilist/build.gradle.kts index b66152d1..7b329823 100644 --- a/anilist/build.gradle.kts +++ b/anilist/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt index cba344eb..bf65a1cb 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt @@ -11,7 +11,8 @@ import java.net.URI */ public object AnilistConfig : MetaDataProviderConfig { - override fun buildDataDownloadLink(id: String): URI = URI("https://${hostname()}/graphql") + // Official public GraphQL API. The anime itself is still linked on the website (hostname). + override fun buildDataDownloadLink(id: String): URI = URI("https://graphql.anilist.co") override fun hostname(): Hostname = "anilist.co" diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt deleted file mode 100644 index 0a1a7a04..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt +++ /dev/null @@ -1,21 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank - -/** - * Central place to store the current anilist token so that the same token is always being used. - * @since 1.0.0 - */ -public object AnilistDefaultTokenRepository : AnilistTokenRepository { - - private var currentToken = AnilistToken(EMPTY, EMPTY) - - override var token: AnilistToken - get() = currentToken - set(value) { - require(value.cookie.neitherNullNorBlank()) { "Cookie must not blank" } - require(value.csrfToken.neitherNullNorBlank()) { "CSRF token must not blank" } - currentToken = value - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt deleted file mode 100644 index ffebed23..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt +++ /dev/null @@ -1,83 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_CPU -import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK -import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank -import io.github.manamiproject.modb.core.extensions.remove -import io.github.manamiproject.modb.core.extractor.DataExtractor -import io.github.manamiproject.modb.core.extractor.XmlDataExtractor -import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.logging.LoggerDelegate -import kotlinx.coroutines.withContext - -private const val CSRF_TOKEN_PREFIX = "window.al_token" - -/** - * Retrieves a valid token. - * @since 1.0.0 - * @property metaDataProviderConfig Configuration for retrieving the token. - * @property httpClient To download the site from which the token will be extracted - * @property extractor Extractor which retrieves the data from raw data. - */ -public class AnilistDefaultTokenRetriever( - private val metaDataProviderConfig: MetaDataProviderConfig = AnilistDefaultTokenRetrieverConfig, - private val httpClient: HttpClient = DefaultHttpClient(isTestContext=metaDataProviderConfig.isTestContext()), - private val extractor: DataExtractor = XmlDataExtractor, -): AnilistTokenRetriever { - - override suspend fun retrieveToken(): AnilistToken = withContext(LIMITED_NETWORK) { - log.info { "Fetching token for anilist." } - - return@withContext httpClient.get( - url = metaDataProviderConfig.buildDataDownloadLink().toURL(), - ).use { response -> - val cookie = extractCookie(response) - val csrfToken = extractCsrfToken(response) - - AnilistToken( - cookie = cookie, - csrfToken = csrfToken, - ) - } - } - - private fun extractCookie(response: HttpResponse): String { - return response.headers["set-cookie"] - ?.map { it.split(';') } - ?.map { it.first() } - ?.sorted() - ?.joinToString("; ") ?: throw IllegalStateException("Unable to extract cookie.") - } - - private suspend fun extractCsrfToken(response: HttpResponse): String = withContext(LIMITED_CPU) { - val responseBody = response.bodyAsString() - require(responseBody.neitherNullNorBlank()) { "Response body must not be empty" } - - val data = extractor.extract(responseBody, selection = mapOf( - "script" to "//script[contains(node(), $CSRF_TOKEN_PREFIX)]/node()" - )) - - return@withContext if (data.notFound("script")) { - throw IllegalStateException("Unable to extract CSRF token.") - } else { - data.listNotNull("script") - .first { it.startsWith(CSRF_TOKEN_PREFIX) } - .remove("$CSRF_TOKEN_PREFIX = \"") - .remove("\";") - .trim() - } - } - - public companion object { - private val log by LoggerDelegate() - - /** - * Singleton of [AnilistDefaultTokenRetriever] - * @since 6.1.0 - */ - public val instance: AnilistDefaultTokenRetriever by lazy { AnilistDefaultTokenRetriever() } - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt deleted file mode 100644 index f9fa548e..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt +++ /dev/null @@ -1,13 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import java.net.URI - -/** - * Configuration for retrieving the CSRF token - * @since 1.0.0 - */ -public object AnilistDefaultTokenRetrieverConfig : MetaDataProviderConfig by AnilistConfig { - - override fun buildDataDownloadLink(id: String): URI = URI("https://${hostname()}") -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt index 5847847b..9ee87257 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt @@ -6,6 +6,7 @@ import io.github.manamiproject.modb.core.downloader.Downloader import io.github.manamiproject.modb.core.extensions.EMPTY import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank import io.github.manamiproject.modb.core.httpclient.APPLICATION_JSON +import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.httpclient.RequestBody import io.github.manamiproject.modb.core.loadResource @@ -20,7 +21,7 @@ import kotlinx.coroutines.runBlocking */ public class AnilistDownloader( private val metaDataProviderConfig: MetaDataProviderConfig = AnilistConfig, - private val httpClient: HttpClient = AnilistHttpClient(), + private val httpClient: HttpClient = DefaultHttpClient(isTestContext = metaDataProviderConfig.isTestContext()), ) : Downloader { private val requestBody: String by lazy { runBlocking { loadResource("anime_download_request.graphql") } } @@ -35,7 +36,6 @@ public class AnilistDownloader( val requestHeaders = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = metaDataProviderConfig.buildAnimeLink(id).toURL(), ) val response = httpClient.post( diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt index a3bd7cf0..f24047e4 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt @@ -1,39 +1,24 @@ package io.github.manamiproject.modb.anilist import io.github.manamiproject.modb.core.httpclient.RequestBody -import java.net.URI -import java.net.URL /** - * Creates all necessary header parameter for an anilist GraphQL request. + * Creates all necessary header parameter for a request to the official anilist GraphQL API. * @since 1.0.0 */ public object AnilistHeaderCreator { /** - * Creates all headers necessary for an anilist GraphQL request. + * Creates all headers necessary for a request to the official anilist GraphQL API. + * The public API doesn't require a session cookie or csrf token. * @since 1.0.0 - * @param requestBody The request body being sent. It's necessary for content-length parameter. - * @param referer The referer [URL]. **Default**: _https://anilist.co_ + * @param requestBody The request body being sent. * @return A [Map] of header parameter. Key is the name of the parameter itself and value is a list of values for this specific header parameter. */ - public fun createAnilistHeaders(requestBody: RequestBody, referer: URL = URI("https://anilist.co").toURL()): Map> { - val url = "https://anilist.co" - + public fun createAnilistHeaders(requestBody: RequestBody): Map> { return mapOf( - "authority" to listOf(url), - "method" to listOf("POST"), - "path" to listOf("/graphql"), - "scheme" to listOf("https"), - "accept" to listOf("*/*"), - "accept-language" to listOf("en-US;q=0.9", "en;q=0.8"), - "content-length" to listOf(requestBody.body.length.toString()), + "accept" to listOf("application/json"), "content-type" to listOf(requestBody.mediaType), - "origin" to listOf(url), - "schema" to listOf("default"), - "cookie" to listOf(AnilistDefaultTokenRepository.token.cookie), - "x-csrf-token" to listOf(AnilistDefaultTokenRepository.token.csrfToken), - "referer" to listOf(referer.toString()) ) } -} \ No newline at end of file +} diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt deleted file mode 100644 index f8584851..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt +++ /dev/null @@ -1,67 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.httpclient.RequestBody -import io.github.manamiproject.modb.core.logging.LoggerDelegate -import java.net.URL - -/** - * Anilist specific implementation of a [HttpClient] which handles the renewal of the token under the hood. - * @since 5.1.1 - * @property delegate HttpClient which is used under the hood. - * @property anilistTokenRetriever Fetches a new token for the CSRF. - * @property anilistTokenRepository Keeps the current CSRF token in-memory. - */ -public class AnilistHttpClient( - private val delegate: HttpClient = DefaultHttpClient(), - private val anilistTokenRetriever: AnilistTokenRetriever = AnilistDefaultTokenRetriever.instance, - private val anilistTokenRepository: AnilistTokenRepository = AnilistDefaultTokenRepository, -): HttpClient by delegate { - - override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { - val initialResponse = delegate.post(url, requestBody, headers) - - if (initialResponse.code != 403) { - return initialResponse - } - - initialResponse.close() - return delegate.post(url, requestBody, renewTokenInHeaders(headers)) - } - - override suspend fun get(url: URL, headers: Map>): HttpResponse { - val initialResponse = delegate.get(url, headers) - - if (initialResponse.code != 403) { - return initialResponse - } - - initialResponse.close() - return delegate.get(url, renewTokenInHeaders(headers)) - } - - private suspend fun renewTokenInHeaders(headers: Map>): Map> { - log.warn { "Anilist responds with 403. Refreshing token." } - anilistTokenRepository.token = anilistTokenRetriever.retrieveToken() - val modifiedHeaders = HashMap(headers).apply { - put("cookie", listOf(AnilistDefaultTokenRepository.token.cookie)) - put("x-csrf-token", listOf(AnilistDefaultTokenRepository.token.csrfToken)) - } - log.info { "Token has been renewed" } - - return modifiedHeaders - } - - - public companion object { - private val log by LoggerDelegate() - - /** - * Singleton of [AnilistHttpClient] - * @since 1.0.0 - */ - public val instance: AnilistHttpClient by lazy { AnilistHttpClient() } - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt deleted file mode 100644 index 8bd83a8d..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt +++ /dev/null @@ -1,12 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Contains the CSRF token and the corresponding cookie value. - * @since 1.0.0 - * @property cookie Value for cookies header parameter - * @property csrfToken Value for CSRF token header parameter - */ -public data class AnilistToken ( - val cookie: String, - val csrfToken: String, -) \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt deleted file mode 100644 index 651ecadc..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt +++ /dev/null @@ -1,16 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Stores the current anilist token. - * @since 2.0.0 - */ -public interface AnilistTokenRepository { - - /** - * Currently valid token or a token with empty fields after initialization. - * The token is only accepted if both cookie and csrf field are not blank. - * @since 1.0.0 - * @throws IllegalArgumentException if the given token contains blank fields - */ - public var token: AnilistToken -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt deleted file mode 100644 index b704013b..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt +++ /dev/null @@ -1,14 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Retrieves the [AnilistToken] which is needed to query the GraphQL endpoints. - * @since 1.0.0 - */ -public interface AnilistTokenRetriever { - - /** - * @since 5.0.0 - * @return Valid [AnilistToken] which is needed to download data - */ - public suspend fun retrieveToken(): AnilistToken -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt index 31cbe5dc..6c102066 100644 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt +++ b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt @@ -45,7 +45,7 @@ internal class AnilistConfigTest { val result = AnilistConfig.buildDataDownloadLink(id) // then - assertThat(result).isEqualTo(URI("https://anilist.co/graphql")) + assertThat(result).isEqualTo(URI("https://graphql.anilist.co")) } @Test diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt deleted file mode 100644 index 31b5e27e..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt +++ /dev/null @@ -1,65 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import org.assertj.core.api.Assertions.assertThat -import kotlin.test.Test -import org.junit.jupiter.api.assertThrows - -internal class AnilistDefaultTokenRepositoryTest { - - @Test - fun `successfully set token`() { - // given - val token = AnilistToken("my-cookie-value", "my-csrf-token-value") - - // when - AnilistDefaultTokenRepository.token = token - - // then - assertThat(AnilistDefaultTokenRepository.token).isEqualTo(token) - } - - @Test - fun `throws exception if cookie is blank`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken(" ", "test") - } - - // then - assertThat(result).hasMessage("Cookie must not blank") - } - - @Test - fun `throws exception if cookie is empty`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken(EMPTY, "test") - } - - // then - assertThat(result).hasMessage("Cookie must not blank") - } - - @Test - fun `throws exception if csrf token is blank`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken("test", " ") - } - - // then - assertThat(result).hasMessage("CSRF token must not blank") - } - - @Test - fun `throws exception if csrf token is empty`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken("test", EMPTY) - } - - // then - assertThat(result).hasMessage("CSRF token must not blank") - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt deleted file mode 100644 index e1f20df4..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import org.assertj.core.api.Assertions.assertThat -import java.net.URI -import kotlin.test.Test - -internal class AnilistDefaultTokenRetrieverConfigTest { - - @Test - fun `hostname must be correct`() { - // when - val result = AnilistDefaultTokenRetrieverConfig.hostname() - - // then - assertThat(result).isEqualTo("anilist.co") - } - - @Test - fun `build anime link correctly`() { - // given - val id = "4563" - - // when - val result = AnilistDefaultTokenRetrieverConfig.buildAnimeLink(id) - - // then - assertThat(result).isEqualTo(URI("https://anilist.co/anime/4563")) - } - - @Test - fun `build data download link correctly`() { - // given - val type = "music" - - // when - val result = AnilistDefaultTokenRetrieverConfig.buildDataDownloadLink(type) - - // then - assertThat(result).isEqualTo(URI("https://anilist.co")) - } - - @Test - fun `file suffix must be html`() { - // when - val result = AnilistDefaultTokenRetrieverConfig.fileSuffix() - - // then - assertThat(result).isEqualTo("json") - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt deleted file mode 100644 index fb861dad..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt +++ /dev/null @@ -1,205 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.client.WireMock.* -import io.github.manamiproject.modb.core.config.FileSuffix -import io.github.manamiproject.modb.core.config.Hostname -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import io.github.manamiproject.modb.core.httpclient.APPLICATION_JSON -import io.github.manamiproject.modb.test.MockServerTestCase -import io.github.manamiproject.modb.test.WireMockServerCreator -import io.github.manamiproject.modb.test.exceptionExpected -import io.github.manamiproject.modb.test.loadTestResource -import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Nested -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ValueSource -import java.net.URI -import kotlin.test.Test - -internal class AnilistDefaultTokenRetrieverTest : MockServerTestCase by WireMockServerCreator() { - - @ParameterizedTest - @ValueSource(strings = [ - "", - " ", - "\u00A0", - "\u202F", - "\u200A", - "\u205F", - "\u2000", - "\u2001", - "\u2002", - "\u2003", - "\u2004", - "\u2005", - "\u2006", - "\u2007", - "\u2008", - "\u2009", - "\uFEFF", - "\u180E", - "\u2060", - "\u200D", - "\u0090", - "\u200C", - "\u200B", - "\u00AD", - "\u000C", - "\u2028", - "\r", - "\n", - "\t", - ]) - fun `throws exception if the response body is blank`(value: String) { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly" - ) - .withStatus(200) - .withBody(value) - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Response body must not be empty") - } - - @Test - fun `throws exception if the csrf token cannot be retrieved`() { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly") - .withStatus(200) - .withBody("response body without token") - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Unable to extract CSRF token.") - } - - @Test - fun `throw exception if the cookie cannot be retrieved`() { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - val responseBody = loadTestResource("AnilistDefaultTokenRetrieverTest/page_containing_token.html") - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withStatus(200) - .withBody(responseBody) - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Unable to extract cookie.") - } - - @Test - fun `correctly retrieve token`() { - runTest { - // given - val testAnilistConfig = object : MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - val responseBody = loadTestResource("AnilistDefaultTokenRetrieverTest/page_containing_token.html") - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly" - ) - .withStatus(200) - .withBody(responseBody) - ) - ) - - // when - val result = anilistTokenRetriever.retrieveToken() - - // then - assertThat(result.cookie).isEqualTo("__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL") - assertThat(result.csrfToken).isEqualTo("IAasRzCsdYp2b5QWQEWtMzSvDzf8UboK0GiH907Y") - } - } - - @Nested - inner class CompanionObjectTests { - - @Test - fun `instance property always returns same instance`() { - // given - val previous = AnilistDefaultTokenRetriever.instance - - // when - val result = AnilistDefaultTokenRetriever.instance - - // then - assertThat(result).isExactlyInstanceOf(AnilistDefaultTokenRetriever::class.java) - assertThat(result === previous).isTrue() - } - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt index a3422f75..a65db77a 100644 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt +++ b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt @@ -2,16 +2,13 @@ package io.github.manamiproject.modb.anilist import io.github.manamiproject.modb.core.httpclient.RequestBody import org.assertj.core.api.Assertions.assertThat -import java.net.URI import kotlin.test.Test internal class AnilistHeaderCreatorTest { @Test - fun `default referer`() { + fun `creates the headers for the official API`() { // given - AnilistDefaultTokenRepository.token = AnilistToken("cookie-value", "csrf-token-value") - val requestBody = RequestBody( mediaType = "used media type", body = "payload", @@ -23,23 +20,13 @@ internal class AnilistHeaderCreatorTest { ) // then - assertThat(result["authority"]).isEqualTo(listOf("https://anilist.co")) - assertThat(result["method"]).isEqualTo(listOf("POST")) - assertThat(result["path"]).isEqualTo(listOf("/graphql")) - assertThat(result["scheme"]).isEqualTo(listOf("https")) - assertThat(result["accept"]).isEqualTo(listOf("*/*")) - assertThat(result["accept-language"]).isEqualTo(listOf("en-US;q=0.9", "en;q=0.8")) - assertThat(result["content-length"]).isEqualTo(listOf("7")) + assertThat(result).containsOnlyKeys("accept", "content-type") + assertThat(result["accept"]).isEqualTo(listOf("application/json")) assertThat(result["content-type"]).isEqualTo(listOf("used media type")) - assertThat(result["cookie"]).isEqualTo(listOf("cookie-value")) - assertThat(result["origin"]).isEqualTo(listOf("https://anilist.co")) - assertThat(result["schema"]).isEqualTo(listOf("default")) - assertThat(result["x-csrf-token"]).isEqualTo(listOf("csrf-token-value")) - assertThat(result["referer"]).isEqualTo(listOf("https://anilist.co")) } @Test - fun `override referer`() { + fun `doesn't send session cookie or csrf token`() { // given val requestBody = RequestBody( mediaType = "used media type", @@ -49,10 +36,9 @@ internal class AnilistHeaderCreatorTest { // when val result = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = URI("https://anilist.co/anime/1535").toURL(), ) // then - assertThat(result["referer"]).isEqualTo(listOf("https://anilist.co/anime/1535")) + assertThat(result).doesNotContainKeys("cookie", "x-csrf-token") } -} \ No newline at end of file +} diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt deleted file mode 100644 index 83375b67..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt +++ /dev/null @@ -1,388 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.httpclient.RequestBody -import io.github.manamiproject.modb.test.shouldNotBeInvoked -import io.github.manamiproject.modb.test.tempDirectory -import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Nested -import java.net.URI -import java.net.URL -import kotlin.test.Test - -internal class AnilistHttpClientTest { - - @Nested - inner class GetTests { - - @Test - fun `successfully loads an entry and retrieves a token upon first call`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - } - } - - var tokenHasBeenSet = false - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { tokenHasBeenSet = true } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(tokenHasBeenSet).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - - @Test - fun `updates token if http response code is 403`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "updated-cookie-value", - csrfToken = "updated-csrf-value", - ) - } - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - private var currentToken = AnilistToken("initial-cookie-value", "initial-csrf-token") - - override var token: AnilistToken - get() = currentToken - set(value) { currentToken = value } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - assertThat(testAnilistDefaultTokenRepository.token.cookie).isEqualTo("updated-cookie-value") - assertThat(testAnilistDefaultTokenRepository.token.csrfToken).isEqualTo("updated-csrf-value") - } - } - - @Test - fun `doesn't trigger token renewal if status code is 200`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return HttpResponse( - code = 200, - body = "success", - ) - } - } - - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken = shouldNotBeInvoked() - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { shouldNotBeInvoked() } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(delegateCallTimes).isEqualTo(1) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - } - - @Nested - inner class PostTests { - - @Test - fun `successfully loads an entry and retrieves a token upon first call`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - } - } - - var tokenHasBeenSet = false - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { tokenHasBeenSet = true } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(tokenHasBeenSet).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - - @Test - fun `updates token if http response code is 403`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "updated-cookie-value", - csrfToken = "updated-csrf-value", - ) - } - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - private var currentToken = AnilistToken("initial-cookie-value", "initial-csrf-token") - - override var token: AnilistToken - get() = currentToken - set(value) { currentToken = value } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - assertThat(testAnilistDefaultTokenRepository.token.cookie).isEqualTo("updated-cookie-value") - assertThat(testAnilistDefaultTokenRepository.token.csrfToken).isEqualTo("updated-csrf-value") - } - } - - @Test - fun `doesn't trigger token renewal if status code is 200`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return HttpResponse( - code = 200, - body = "success", - ) - } - } - - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken = shouldNotBeInvoked() - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { shouldNotBeInvoked() } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(delegateCallTimes).isEqualTo(1) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - } - - @Nested - inner class CompanionObjectTests { - - @Test - fun `instance property always returns same instance`() { - tempDirectory { - // given - val previous = AnilistHttpClient.instance - - // when - val result = AnilistHttpClient.instance - - // then - assertThat(result).isExactlyInstanceOf(AnilistHttpClient::class.java) - assertThat(result === previous).isTrue() - } - } - } -} diff --git a/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html b/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html deleted file mode 100644 index 1d3fd749..00000000 --- a/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html +++ /dev/null @@ -1,66 +0,0 @@ - - -AniList - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - \ No newline at end of file diff --git a/anime-planet/build.gradle.kts b/anime-planet/build.gradle.kts index 825451a3..898add7d 100644 --- a/anime-planet/build.gradle.kts +++ b/anime-planet/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/animenewsnetwork/build.gradle.kts b/animenewsnetwork/build.gradle.kts index 55d80505..5b03c8fe 100644 --- a/animenewsnetwork/build.gradle.kts +++ b/animenewsnetwork/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anisearch/build.gradle.kts b/anisearch/build.gradle.kts index fe6c54cc..56c42900 100644 --- a/anisearch/build.gradle.kts +++ b/anisearch/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8aa7f756..f85da332 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -50,6 +50,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -63,12 +65,19 @@ application { } tasks { + // Bundled so that the app can offer to create a config.toml from it on the first start. + named("processResources") { + from(rootProject.file("config.example.toml")) + } + named("shadowJar") { archiveBaseName.set("modb-app") archiveClassifier.set("") archiveVersion.set("") manifest { attributes["Main-Class"] = mainClassPath + // zstd-jni loads a native library, which JDK 24+ only allows silently with this attribute. + attributes["Enable-Native-Access"] = "ALL-UNNAMED" } exclude(".gitemptydir") } diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 8a260d99..78382a89 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -1,5 +1,8 @@ package io.github.manamiproject.modb.app +import io.github.manamiproject.modb.anidb.AnidbBannedException +import io.github.manamiproject.modb.anidb.CrawlerDetectedException +import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig import io.github.manamiproject.modb.app.convfiles.DefaultRawFileConversionService @@ -21,6 +24,7 @@ import io.github.manamiproject.modb.app.downloadcontrolstate.DefaultDownloadCont import io.github.manamiproject.modb.app.extensions.alertDeletedAnimeByTitle import io.github.manamiproject.modb.app.fluentapi.* import io.github.manamiproject.modb.app.network.LinuxNetworkController +import io.github.manamiproject.modb.app.network.NetworkControllers import io.github.manamiproject.modb.app.postprocessors.* import io.github.manamiproject.modb.core.coroutines.CoroutineManager.runCoroutine import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK @@ -33,17 +37,39 @@ import javax.swing.JPasswordField import javax.swing.SwingUtilities import kotlin.system.exitProcess +// Lazy, because the first logger sets up logback, which reads the configuration. That has to wait +// until ensureConfigFileExists() had the chance to create it. @KoverIgnore -private val log = org.slf4j.LoggerFactory.getLogger("App") +private val log by lazy { org.slf4j.LoggerFactory.getLogger("App") } -fun main() = runCoroutine { - LinuxNetworkController.instance.sudoPasswordValue = passwordPrompt() +fun main() { + ensureConfigFileExists() + crawlAndCreateDataset() +} + +private fun crawlAndCreateDataset() = runCoroutine { + val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() + + // Only the crawlers of these sites change the outgoing address by restarting the network device. + // Both aniSearch crawlers use the hostname of anisearch.com, but can be disabled independently. + val sitesChangingAddress = buildSet { + if (AnidbConfig.hostname() !in disabled) { + add(AnidbConfig.hostname()) + } + if (AnisearchConfig.hostname() !in disabled || ANISEARCH_RELATIONS_CRAWLER !in disabled) { + add(AnisearchConfig.hostname()) + } + } + + when { + isRoot() -> log.info("Running as root. No sudo password needed.") + !NetworkControllers.usesNetworkDevice(sitesChangingAddress) -> log.info("Network device won't be restarted. No sudo password needed.") + else -> LinuxNetworkController.instance.sudoPasswordValue = verifiedSudoPassword() + } val rawFileConversionService = DefaultRawFileConversionService.instance rawFileConversionService.start() - val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() - val crawlers = mutableListOf() withContext(LIMITED_NETWORK) { @@ -52,7 +78,7 @@ fun main() = runCoroutine { crawl("anime-planet.com", disabled, crawlers) { AnimePlanetCrawler.instance.start() } crawl("animenewsnetwork.com", disabled, crawlers) { AnimenewsnetworkCrawler.instance.start() } crawl("anisearch.com", disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchConfig).start() } - crawl("anisearch.com-relations", disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchRelationsConfig).start() } + crawl(ANISEARCH_RELATIONS_CRAWLER, disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchRelationsConfig).start() } crawl("kitsu.app", disabled, crawlers) { KitsuCrawler.instance.start() } crawl("livechart.me", disabled, crawlers) { LivechartCrawler.instance.start() } crawl("myanimelist.net", disabled, crawlers) { MyanimelistCrawler.instance.start() } @@ -91,6 +117,48 @@ fun main() = runCoroutine { ).forEach { it.process() } } +@KoverIgnore +private fun isRoot(): Boolean = try { + val process = ProcessBuilder("id", "-u").redirectErrorStream(true).start() + val userId = process.inputStream.bufferedReader().readText().trim() + process.waitFor() == 0 && userId == "0" +} catch (_: Exception) { + false +} + +@KoverIgnore +private fun verifiedSudoPassword(): String { + repeat(MAX_SUDO_ATTEMPTS) { attempt -> + val password = passwordPrompt() + + if (isValidSudoPassword(password)) { + return password + } + + log.warn("Incorrect sudo password [${attempt + 1}/$MAX_SUDO_ATTEMPTS].") + } + + log.error("No valid sudo password after [$MAX_SUDO_ATTEMPTS] attempts. Exiting.") + exitProcess(1) +} + +/** + * `-k` ignores cached credentials so that the password is really checked, `-S` reads it from stdin + * and `-v` only validates without running a command. + */ +@KoverIgnore +private fun isValidSudoPassword(password: String): Boolean = try { + val process = ProcessBuilder("sudo", "-k", "-S", "-p", "", "-v").redirectErrorStream(true).start() + process.outputStream.bufferedWriter().use { it.write(password); it.newLine() } + process.inputStream.bufferedReader().readText() + process.waitFor() == 0 +} catch (_: Exception) { + false +} + +private const val MAX_SUDO_ATTEMPTS = 3 +private const val ANISEARCH_RELATIONS_CRAWLER = "anisearch.com-relations" + @KoverIgnore private fun passwordPrompt(): String { val console = System.console() @@ -143,8 +211,17 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: jobs.add(launch { try { block.invoke() + } catch (_: AnidbBannedException) { + // Expected while a ban lasts, so a stack trace adds nothing. + log.error("Crawler for [$hostname] stopped, because AniDB still answers with [banned] after all retries. Wait until the ban has been lifted or add [$hostname] to [modb.app.disabledCrawlers].") + } catch (_: CrawlerDetectedException) { + // AniDB refusing the connection is an answer, not a fault in the code, so a stack trace adds nothing. + log.error("Crawler for [$hostname] stopped, because AniDB refuses the connection (AntiLeech or 403). Try again later or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { - log.error("Crawler for [$hostname] stopped: ${e.message}", e) + // The message says what happened. The stack trace is only of use when debugging, so it is logged at trace + // level, which the regular log levels (down to debug) leave out. + log.error("Crawler for [$hostname] stopped: ${e.message ?: e::class.simpleName}") + log.trace("Stack trace of the stopped crawler for [$hostname]", e) } }) } diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt new file mode 100644 index 00000000..c2824afb --- /dev/null +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt @@ -0,0 +1,100 @@ +package io.github.manamiproject.modb.app + +import io.github.manamiproject.modb.core.coverage.KoverIgnore +import io.github.manamiproject.modb.serde.json.serializer.DeadEntriesJsonSerializer +import kotlinx.coroutines.runBlocking +import java.nio.file.Path +import kotlin.io.path.Path +import kotlin.io.path.absolute +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.writeText +import kotlin.system.exitProcess + +private const val CONFIG_FILE = "config.toml" +private const val CONFIG_LOCATION_KEY = "modb.core.config.location" +private const val TEMPLATE_RESOURCE = "config.example.toml" +private val DEAD_ENTRIES_FILES = listOf("anidb", "anilist", "animenewsnetwork", "kitsu", "myanimelist") + +/** + * Offers to create a `config.toml` from the template if none can be found. + * + * Has to run before anything logs: logback determines its log directory from the configuration, and + * the configuration is read only once, so a file created afterwards would not be seen. + */ +@KoverIgnore +internal fun ensureConfigFileExists() { + if (configFileExists()) { + return + } + + val workingDirectory = Path("").absolute() + val configFile = workingDirectory.resolve(CONFIG_FILE) + + println("No [$CONFIG_FILE] found in [$workingDirectory].") + println("Either start the app from the folder containing your [$CONFIG_FILE] or pass its location with -D$CONFIG_LOCATION_KEY=/path/to/$CONFIG_FILE.") + println() + println("Create [$configFile] from the default template instead?") + println(" downloadsDirectory = [${workingDirectory.resolve("downloads")}]") + println(" outputDirectory = [${workingDirectory.resolve("output")}]") + println(" downloadControlStateDirectory = [${workingDirectory.resolve("dcs")}]") + println(" disabledCrawlers = [anidb.net]") + print("Create it? [y/N]: ") + System.out.flush() + + val answer = readlnOrNull()?.trim()?.lowercase() + + if (answer != "y" && answer != "yes") { + println("No config created. Exiting.") + exitProcess(1) + } + + createDefaultConfig(workingDirectory, configFile) + println("Created [$configFile]. Add your AniDB client to it before enabling the anidb.net crawler.") + println() +} + +private fun configFileExists(): Boolean { + val overriddenLocation = System.getProperty(CONFIG_LOCATION_KEY) ?: System.getenv(CONFIG_LOCATION_KEY) + + // An explicitly given location is the caller's choice. If it is wrong, the config registry reports it. + if (!overriddenLocation.isNullOrBlank()) { + return true + } + + return Path(CONFIG_FILE).exists() || Thread.currentThread().contextClassLoader.getResource(CONFIG_FILE) != null +} + +private fun createDefaultConfig(workingDirectory: Path, configFile: Path) { + val downloads = workingDirectory.resolve("downloads").createDirectories() + val output = workingDirectory.resolve("output").createDirectories() + val dcs = workingDirectory.resolve("dcs").createDirectories() + + val template = Thread.currentThread().contextClassLoader.getResourceAsStream(TEMPLATE_RESOURCE) + ?.bufferedReader() + ?.use { it.readText() } + ?: error("Template [$TEMPLATE_RESOURCE] is missing from the app.") + + val config = template + .setValue("downloadsDirectory", downloads) + .setValue("outputDirectory", output) + .setValue("downloadControlStateDirectory", dcs) + + configFile.writeText(config) + + // The dataset expects a dead entries file for each provider which has one, even on the very first run. + val deadEntriesDirectory = output.resolve("dead-entries").createDirectories() + val emptyDeadEntries = runBlocking { DeadEntriesJsonSerializer.instance.serialize(emptyList(), minify = true) } + + DEAD_ENTRIES_FILES + .map { deadEntriesDirectory.resolve("$it-minified.json") } + .filterNot { it.exists() } + .forEach { it.writeText(emptyDeadEntries) } +} + +private fun String.setValue(key: String, directory: Path): String { + val value = directory.toString().replace("\\", "\\\\").replace("\"", "\\\"") + val line = Regex("""(?m)^(\s*$key\s*=\s*)"[^"]*"""") + check(line.containsMatchIn(this)) { "Template doesn't contain [$key]." } + return line.replace(this) { "${it.groupValues[1]}\"$value\"" } +} diff --git a/app/src/main/resources/logback.xml b/app/src/main/resources/logback.xml index 04931f0b..1646138d 100644 --- a/app/src/main/resources/logback.xml +++ b/app/src/main/resources/logback.xml @@ -1,6 +1,6 @@ - + diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 00000000..a7e4e502 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,25 @@ +[modb.app] +downloadsDirectory = "" +outputDirectory = "" +downloadControlStateDirectory = "" +disabledCrawlers = ["anidb.net"] + +[modb.app.challenge] +solverUrl = "http://localhost:8191/v1" + +[modb.app.crawler] +# Requested again per run: dead entries of AniList, Kitsu, MAL (highest IDs first), pending entries of AniDB (longest unchecked first). +deadEntriesToRecheck = 0 + +[modb.core.httpclient] +# Retries after server errors, 429 and timeouts, for all sites. +maxRetries = 5 + +[modb.anidb] +# api | website +source = "api" +# Client registered for the AniDB HTTP API. +client = "" +clientver = "" +# Retries while AniDB answers "banned", before the AniDB crawler stops. +banRetries = 5 diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 9d10e14f..c642ccee 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -61,6 +61,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -68,12 +70,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt index 801ec705..e7c0a070 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt @@ -114,7 +114,7 @@ internal object JsoupCssSelectorDataExtractor : DataExtractor { terminatingChild == "text()" -> jsoupElements.eachText() terminatingChild == "node()" -> jsoupElements.dataNodes().map { it.wholeData.trim() } terminatingChild.eitherNullOrBlank() -> jsoupElements.textNodes().map { it.text().trim() }.filter { it.neitherNullNorBlank() } - terminatingChild.startsWith('@') -> jsoupElements.eachAttr(terminatingChild.trimStart('@')) ?: NotFound + terminatingChild.startsWith('@') -> jsoupElements.eachAttr(terminatingChild.trimStart('@')) else -> throw IllegalStateException("unmapped case") } } diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt index e3e40952..32b0b620 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt @@ -54,10 +54,10 @@ internal object JsoupXPathDataExtractor : DataExtractor { val value = when { jsoupElements.isEmpty() -> NotFound - entry.third == "text()" -> jsoupElements.eachText() ?: NotFound + entry.third == "text()" -> jsoupElements.eachText() entry.third == "node()" -> jsoupElements.dataNodes().map { it.wholeData.trim() } entry.third.eitherNullOrBlank() -> jsoupElements.textNodes().map { it.text().trim() }.filter { it.neitherNullNorBlank() } - else -> jsoupElements.eachAttr(entry.third) ?: NotFound + else -> jsoupElements.eachAttr(entry.third) } return entry.first to value diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt new file mode 100644 index 00000000..1eba3b86 --- /dev/null +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt @@ -0,0 +1,33 @@ +package io.github.manamiproject.modb.core.httpclient + +/** + * Cloudflare answers a client it doesn't trust (yet) with an interstitial page instead of the content: the browser + * check "Just a moment..." or the block page "Attention Required!". Depending on the site this comes with 403, 503, + * 429 or even 200. + * + * Reads the body, which [HttpResponse] keeps in memory afterwards, so the caller can still read it. + * @since 1.0.0 + * @return `true` if the response is one of Cloudflare's interstitial pages rather than the requested content. + */ +public fun HttpResponse.isCloudflareChallenge(): Boolean { + if (code !in CHALLENGE_CODES) { + return false + } + + val body = bodyAsString() + return CHALLENGE_MARKERS.any { body.contains(it, ignoreCase = true) } +} + +/** + * Retries a request answered with a Cloudflare interstitial page, waiting like any other retry. + * @since 1.0.0 + */ +public val CloudflareChallengeRetryCase: HttpResponseRetryCase = HttpResponseRetryCase { it.isCloudflareChallenge() } + +private val CHALLENGE_CODES = setOf(200, 403, 429, 503) + +private val CHALLENGE_MARKERS = listOf( + "Just a moment...", + "Attention Required! | Cloudflare", + "window._cf_chl_opt", +) diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index cb559bf2..0e1c9819 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -1,5 +1,8 @@ package io.github.manamiproject.modb.core.httpclient +import io.github.manamiproject.modb.core.config.ConfigRegistry +import io.github.manamiproject.modb.core.config.DefaultConfigRegistry +import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK import io.github.manamiproject.modb.core.coverage.KoverIgnore import io.github.manamiproject.modb.core.extensions.EMPTY @@ -12,6 +15,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.withContext +import kotlin.time.Duration import okhttp3.* import okhttp3.Protocol.HTTP_1_1 as OKHTTP_HTTP_1_1 import okhttp3.Protocol.HTTP_2 as OKHTTP_HTTP_2 @@ -52,6 +56,7 @@ import java.util.concurrent.TimeUnit.* * @property okhttpClient Instance of the OKHTTP client on which this client is based. * @property isTestContext Whether this runs in the unit test context or not. * @property headerCreator Creates default headers based on the selected browser type. + * @param configRegistry Source of `modb.core.httpclient.maxRetries`, the number of retries before a request fails. **Default:** `5` */ public class DefaultHttpClient( proxy: Proxy = NO_PROXY, @@ -70,9 +75,17 @@ public class DefaultHttpClient( private val isTestContext: Boolean = false, private val headerCreator: HeaderCreator = DefaultHeaderCreator.instance, useDefaultRetryCases: Boolean = true, + configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ) : HttpClient { - private val retryBehavior: RetryBehavior = RetryBehavior() + private val maxRetries: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = CONFIG_NAMESPACE, + default = DEFAULT_MAX_RETRIES, + validator = { it >= 0 }, + ) + + private val retryBehavior: RetryBehavior = RetryBehavior(maxAttempts = maxRetries) init { if (useDefaultRetryCases) retryBehavior.addCases( @@ -158,6 +171,8 @@ public class DefaultHttpClient( var attempt = 0 var retryCase: RetryCase = NoRetry var responseOrException: Any = HttpResponse(100, EMPTY) + var retryAfter: Duration? = null + var reason = EMPTY do { if (retryCase !is NoRetry) { @@ -165,7 +180,7 @@ public class DefaultHttpClient( if ((responseOrException is HttpResponse) && (responseOrException.code == 103)) { responseOrException.close() - log.warn { "Received HTTP status code 103. Deactivating HTTP/2." } + log.warn { "Received HTTP status code 103 from [${request.url.host}]. Deactivating HTTP/2." } protocols.remove(HTTP_2) } @@ -185,7 +200,7 @@ public class DefaultHttpClient( okhttpClient = currentClient.newBuilder().build() } - wait(request, retryCase, attempt) + wait(request, retryCase, attempt, retryAfter, reason) } responseOrException = safelyExecute { @@ -201,7 +216,11 @@ public class DefaultHttpClient( true -> { log.debug { "[${request.method} ${request.url}] Requires retry." } + retryAfter = null + reason = reasonOf(responseOrException) + if (responseOrException is HttpResponse) { + retryAfter = parseRetryAfter(responseOrException.headers) responseOrException.close() } @@ -228,11 +247,28 @@ public class DefaultHttpClient( } } + private fun reasonOf(responseOrException: Any): String = when (responseOrException) { + // The body is only looked at if a retry case has read it already. Reading it here would download pages + // which are going to be discarded anyway. + is HttpResponse -> when { + responseOrException.isBodyInputStreamExhausted() && runCatching { responseOrException.isCloudflareChallenge() }.getOrDefault(false) -> { + "HTTP ${responseOrException.code}, Cloudflare challenge" + } + else -> "HTTP ${responseOrException.code}" + } + is Throwable -> listOfNotNull(responseOrException::class.simpleName, responseOrException.message).joinToString(": ") + else -> responseOrException.javaClass.simpleName + } + @KoverIgnore - private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int) { + private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int, retryAfter: Duration?, reason: String) { if (!isTestContext) { - log.trace { "Initiating waiting time for retry of [${request.method} ${request.url}]." } - delay(retryCase.waitDuration(attempt)) + val duration = retryAfter ?: retryCase.waitDuration(attempt) + val askedByServer = if (retryAfter != null) ", as the server asked" else EMPTY + + // A wait can last minutes, so it is logged at info level: without it a waiting crawler looks like a hung one. + log.info { "Retrying [${request.method} ${request.url}] in [${duration.inWholeSeconds}s] (attempt $attempt/${retryBehavior.maxAttempts}) after [$reason]$askedByServer." } + delay(duration) } } @@ -290,6 +326,18 @@ public class DefaultHttpClient( */ public const val DEFAULT_READ_TIMEOUT_IN_SECONDS: Long = 60L + /** + * Number of retries before a request fails, unless `modb.core.httpclient.maxRetries` sets another one. + * @since 15.0.0 + */ + public const val DEFAULT_MAX_RETRIES: Int = 5 + + /** + * Prefix of the properties read by this class. + * @since 15.0.0 + */ + public const val CONFIG_NAMESPACE: String = "modb.core.httpclient" + /** * Singleton of [DefaultHttpClient] * @since 15.0.0 diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt new file mode 100644 index 00000000..36eb5610 --- /dev/null +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt @@ -0,0 +1,40 @@ +package io.github.manamiproject.modb.core.httpclient + +import java.time.Clock +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME +import java.time.format.DateTimeParseException +import kotlin.time.Duration +import kotlin.time.Duration.Companion.ZERO +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +/** + * Reads the `Retry-After` header of a response (RFC 9110, section 10.2.3). + * The value is either a number of seconds or an HTTP date. + * @since 20.0.0 + * @param headers Headers of the response. The header name is matched case-insensitive. + * @param clock Used to calculate the time until an HTTP date. + * @return How long to wait before the next request or `null` if the header is missing or can't be read. + * A date in the past results in [ZERO]. + */ +internal fun parseRetryAfter(headers: Map>, clock: Clock = Clock.systemUTC()): Duration? { + val value = headers.entries + .firstOrNull { it.key.equals("retry-after", ignoreCase = true) } + ?.value + ?.firstOrNull() + ?.trim() + ?: return null + + value.toLongOrNull()?.let { secondsToWait -> + return if (secondsToWait < 0) null else secondsToWait.seconds + } + + return try { + val date = ZonedDateTime.parse(value, RFC_1123_DATE_TIME) + val millisToWait = date.toInstant().toEpochMilli() - clock.millis() + if (millisToWait <= 0) ZERO else millisToWait.milliseconds + } catch (e: DateTimeParseException) { + null + } +} diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt new file mode 100644 index 00000000..cdb71d7e --- /dev/null +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt @@ -0,0 +1,74 @@ +package io.github.manamiproject.modb.core.httpclient + +import org.assertj.core.api.Assertions.assertThat +import kotlin.test.Test + +internal class CloudflareChallengeKtTest { + + private val challenge = """Just a moment...""" + private val content = """All Seasons | LiveChart.me
    """ + + @Test + fun `recognises the browser check`() { + // when + val result = HttpResponse(403, challenge).isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `recognises the browser check served with 200`() { + // when + val result = HttpResponse(200, challenge).isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `recognises the block page`() { + // when + val result = HttpResponse(403, "Attention Required! | Cloudflare").isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `regular content is no challenge`() { + // when + val result = HttpResponse(200, content).isCloudflareChallenge() + + // then + assertThat(result).isFalse() + } + + @Test + fun `an error page of another kind is no challenge`() { + // when + val result = HttpResponse(404, challenge).isCloudflareChallenge() + + // then + assertThat(result).isFalse() + } + + @Test + fun `the body can still be read afterwards`() { + // given + val response = HttpResponse(200, content) + + // when + response.isCloudflareChallenge() + + // then + assertThat(response.bodyAsString()).isEqualTo(content) + } + + @Test + fun `the retry case applies to a challenge only`() { + // then + assertThat(CloudflareChallengeRetryCase.retryIf(HttpResponse(403, challenge))).isTrue() + assertThat(CloudflareChallengeRetryCase.retryIf(HttpResponse(200, content))).isFalse() + } +} diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt index 3bf0867a..106f7a49 100644 --- a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt @@ -4,6 +4,8 @@ import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.matching.EqualToJsonPattern import com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED +import io.github.manamiproject.modb.core.TestConfigRegistry +import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.extensions.EMPTY import io.github.manamiproject.modb.core.httpclient.HttpProtocol.HTTP_1_1 import io.github.manamiproject.modb.test.* @@ -372,6 +374,42 @@ internal class DefaultHttpClientKtTest : MockServerTestCase by W } } + @Test + fun `status code - number of retries can be set via maxRetries`() { + runTest { + // given + val path = "anime/1535" + + serverInstance.stubFor( + get(urlPathEqualTo("/$path")).willReturn( + aResponse() + .withHeader("Content-Type", "text/plain") + .withStatus(429) + .withBody(EMPTY) + ) + ) + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.core.httpclient.maxRetries") 2 else null + } + + val url = URI("http://localhost:$port/$path").toURL() + val client = DefaultHttpClient( + isTestContext = true, + configRegistry = testConfigRegistry, + ) + + // when + val result = exceptionExpected { + client.get(url) + } + + // then + assertThat(result).hasMessage("Execution failed despite [2] retry attempts. Last invocation of [GET http://localhost:$port/anime/1535] returned http status code [429]") + serverInstance.verify(3, getRequestedFor(urlPathEqualTo("/$path"))) + } + } + @Test fun `throwable - exception without a retry case is thrown as-is`() { runTest { diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt new file mode 100644 index 00000000..f286061c --- /dev/null +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt @@ -0,0 +1,77 @@ +package io.github.manamiproject.modb.core.httpclient + +import org.assertj.core.api.Assertions.assertThat +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.test.Test +import kotlin.time.Duration.Companion.ZERO +import kotlin.time.Duration.Companion.seconds + +internal class RetryAfterKtTest { + + private val clock = Clock.fixed(Instant.parse("2026-10-27T20:00:00Z"), UTC) + + @Test + fun `returns null if the header is missing`() { + // when + val result = parseRetryAfter(mapOf("content-type" to listOf("application/json")), clock) + + // then + assertThat(result).isNull() + } + + @Test + fun `reads delay in seconds`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("42")), clock) + + // then + assertThat(result).isEqualTo(42.seconds) + } + + @Test + fun `header name is matched case-insensitive`() { + // when + val result = parseRetryAfter(mapOf("Retry-After" to listOf(" 7 ")), clock) + + // then + assertThat(result).isEqualTo(7.seconds) + } + + @Test + fun `reads http date`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("Tue, 27 Oct 2026 20:01:30 GMT")), clock) + + // then + assertThat(result).isEqualTo(90.seconds) + } + + @Test + fun `http date in the past results in zero`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("Tue, 27 Oct 2026 19:59:00 GMT")), clock) + + // then + assertThat(result).isEqualTo(ZERO) + } + + @Test + fun `returns null for a value which can't be read`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("soon")), clock) + + // then + assertThat(result).isNull() + } + + @Test + fun `returns null for a negative number`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("-5")), clock) + + // then + assertThat(result).isNull() + } +} diff --git a/kitsu/build.gradle.kts b/kitsu/build.gradle.kts index 00e74866..0f089d6e 100644 --- a/kitsu/build.gradle.kts +++ b/kitsu/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/kommand/build.gradle.kts b/kommand/build.gradle.kts index b22a4988..c0bb65e0 100644 --- a/kommand/build.gradle.kts +++ b/kommand/build.gradle.kts @@ -38,17 +38,19 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = Runtime.getRuntime().availableProcessors() } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt index b2699028..00c09fb6 100644 --- a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt +++ b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt @@ -7,6 +7,15 @@ import java.io.IOException private const val SUDO = "sudo" +/** + * How sudo reports a wrong password. Classic sudo says "incorrect password attempt", sudo-rs (the default since + * Ubuntu 25.10) says "Authentication failed". + */ +private val INCORRECT_PASSWORD_MESSAGES = setOf( + "incorrect password attempt", + "authentication failed", +) + /** * If you want to execute a command as `sudo` you can pass `command` in [JavaProcessBuilder.executeCmd] either with or * without sudo as first element. It's only important that [CommandLineConfig.useSudo] is set to `true`. If you don't @@ -55,7 +64,7 @@ public class JavaProcessBuilder( output.addAll(process.inputStream.bufferedReader().readLines()) } - if (config.useSudo && output.any { it.lowercase().contains("incorrect password attempt") }) { + if (config.useSudo && output.any { line -> INCORRECT_PASSWORD_MESSAGES.any { line.lowercase().contains(it) } }) { throw IncorrectPasswordAttemptException } @@ -98,7 +107,8 @@ public class JavaProcessBuilder( } when { - e.cause?.message == "error=2, No such file or directory" -> throw CommandNotFoundException(commandName) + // The wording depends on the JDK, e.g. "error=2, No such file or directory" or "Exec failed, error: 2 (No such file or directory)" + e.cause?.message?.contains("No such file or directory") == true -> throw CommandNotFoundException(commandName) e.cause?.message?.contains("permission denied", false) == true -> throw PermissionDeniedException(commandName) else -> throw e } diff --git a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt index 3edc0682..db2d1a55 100644 --- a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt +++ b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt @@ -5,6 +5,7 @@ import io.github.manamiproject.kommand.exceptions.IncorrectPasswordAttemptExcept import io.github.manamiproject.kommand.exceptions.UnexpectedUsageOfSudo import io.github.manamiproject.kommand.exceptions.UnsupportedOperatingSystemException import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -186,6 +187,8 @@ internal class JavaProcessBuilderTest { @Test fun `throws exception if command with sudo is given an incorrect password`() { + assumeTrue(sudoRequiresPassword(), "sudo doesn't ask for a password on this system, so an incorrect password cannot be detected.") + // given val javaProcessBuilder = JavaProcessBuilder { useSudo = true @@ -219,4 +222,17 @@ internal class JavaProcessBuilderTest { assertThat(result===previous).isTrue() } } -} \ No newline at end of file +} + +private fun sudoRequiresPassword(): Boolean { + return try { + // -n fails instead of asking if a password is required. Without -k, so that credentials cached by an earlier + // sudo in the same terminal count as well: the command under test would use them and never check the password. + ProcessBuilder("sudo", "-n", "true") + .redirectErrorStream(true) + .start() + .waitFor() != 0 + } catch (e: java.io.IOException) { + false + } +} diff --git a/lib/build.gradle.kts b/lib/build.gradle.kts index 173fcd59..6e4e8899 100644 --- a/lib/build.gradle.kts +++ b/lib/build.gradle.kts @@ -63,6 +63,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -70,12 +72,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt index fa20a58f..875f9518 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt @@ -69,6 +69,7 @@ class DependentConversionWatchService( validFileSuffixes.addAll(metaDataConfigs.map { it.value.fileSuffix() }) } workingDirs.values.forEach { deleteLocks(it) } + workingDirs.values.forEach { cleanUpEmptyConvertedFiles(it) } dependentFileConverter.convertUnconvertedFiles() isPrepared = true } @@ -91,6 +92,14 @@ class DependentConversionWatchService( jobs.forEach { it.cancelAndJoin() } } + private suspend fun cleanUpEmptyConvertedFiles(workingDir: Directory) { + val deleted = workingDir.deleteEmptyConvertedFiles() + + if (deleted > 0) { + log.warn { "Deleted [$deleted] empty [*.$CONVERTED_FILE_SUFFIX] files in [$workingDir]. They are converted again." } + } + } + private suspend fun deleteLocks(workingDir: Directory) { log.info { "Deleting [$LOCK_FILE_SUFFIX] files in [$workingDir]" } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt index 2de320b8..9616045e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt @@ -1,7 +1,11 @@ package io.github.manamiproject.modb.app.convfiles +import io.github.manamiproject.modb.core.extensions.Directory import io.github.manamiproject.modb.core.extensions.RegularFile +import io.github.manamiproject.modb.core.extensions.listRegularFiles import io.github.manamiproject.modb.core.anime.AnimeRaw +import kotlin.io.path.deleteIfExists +import kotlin.io.path.fileSize /** * File suffix used for the files containing the serialized [AnimeRaw] object as intermediate format. @@ -9,6 +13,19 @@ import io.github.manamiproject.modb.core.anime.AnimeRaw */ const val CONVERTED_FILE_SUFFIX = "conv" +/** + * Deletes empty converted files. Stopping the app while a file is being converted can leave an empty file behind, + * which would otherwise count as converted. Without it, the raw file is converted again. + * @since 1.0.0 + * @receiver Working directory of a metadata provider. + * @return Number of deleted files. + */ +internal suspend fun Directory.deleteEmptyConvertedFiles(): Int { + return listRegularFiles("*.$CONVERTED_FILE_SUFFIX") + .filter { it.fileSize() == 0L } + .count { it.deleteIfExists() } +} + /** * Converts raw files into a generic intermediate format based on [AnimeRaw]. diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt index 7716a289..c663810c 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt @@ -45,6 +45,7 @@ class SimpleConversionWatchService @KoverIgnore constructor( override suspend fun prepare() = withContext(LIMITED_FS) { deleteLocks() + cleanUpEmptyConvertedFiles() fileConverter.convertUnconvertedFiles() workingDir.register(watchService, ENTRY_CREATE) isPrepared = true @@ -82,6 +83,14 @@ class SimpleConversionWatchService @KoverIgnore constructor( watchService.close() } + private suspend fun cleanUpEmptyConvertedFiles() { + val deleted = workingDir.deleteEmptyConvertedFiles() + + if (deleted > 0) { + log.warn { "Deleted [$deleted] empty [*.$CONVERTED_FILE_SUFFIX] files in [$workingDir]. They are converted again." } + } + } + private suspend fun deleteLocks() { log.info { "Deleting [$LOCK_FILE_SUFFIX] files in [$workingDir]" } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt index 1f08fa7d..b01891ab 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt @@ -28,6 +28,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate * @property downloadControlStateAccessor Access to DCS files. * @property downloadControlStateScheduler Allows to check which anime are scheduled for re-download and which are not. * @property alreadyDownloadedIdsFinder Fetches all IDs which have already been downloaded. + * @property recheckDeadEntries Whether `deadEntriesToRecheck` applies. A crawler which rechecks something else instead turns it off. */ class IntegerBasedIdRangeSelector( private val metaDataProviderConfig: MetaDataProviderConfig, @@ -36,6 +37,7 @@ class IntegerBasedIdRangeSelector( private val downloadControlStateAccessor: DownloadControlStateAccessor = DefaultDownloadControlStateAccessor.instance, private val downloadControlStateScheduler: DownloadControlStateScheduler = DefaultDownloadControlStateScheduler.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, + private val recheckDeadEntries: Boolean = true, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ): IdRangeSelector { @@ -67,23 +69,35 @@ class IntegerBasedIdRangeSelector( // out of the dataset for good. The highest ids are the ones this happens to, because they // are the ones a provider is still filling in, so a slice of them is looked at again. val deadEntries = deadEntriesAccessor.fetchDeadEntries(metaDataProviderConfig).map { it.toInt() }.toSet() - val recheck = deadEntries.sortedDescending().take(deadEntriesToRecheck).toSet() + val recheck = if (recheckDeadEntries) deadEntries.sortedDescending().take(deadEntriesToRecheck).toSet() else emptySet() if (recheck.isNotEmpty()) { log.info { "Rechecking [${recheck.size}] of [${deadEntries.size}] dead entries for [${metaDataProviderConfig.hostname()}]." } } + val beforeDeadEntries = possibleIds.size possibleIds.removeAll(deadEntries - recheck) + val excludedDeadEntries = beforeDeadEntries - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding dead entries." } log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] before excluding entries which are not scheduled for the current week." } + val beforeNotScheduled = possibleIds.size possibleIds.removeAll(downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig).map { it.toInt() }.toSet()) + val excludedNotScheduled = beforeNotScheduled - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding entries which are not scheduled for the current week." } log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] before excluding already downloaded entries." } + val beforeAlreadyDownloaded = possibleIds.size possibleIds.removeAll(alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).map { it.toInt() }.toSet()) + val excludedAlreadyDownloaded = beforeAlreadyDownloaded - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding already downloaded entries." } + log.info { + "[${metaDataProviderConfig.hostname()}] highest ID is [$highestId]. " + + "Skipping [$excludedDeadEntries] dead entries, [$excludedNotScheduled] entries not scheduled for this week " + + "and [$excludedAlreadyDownloaded] already downloaded. [${possibleIds.size}] of [$highestId] IDs left to download." + } + return possibleIds.toList().createShuffledList() } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index fe828a23..b449e4de 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -4,6 +4,9 @@ import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anidb.AnidbApiDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbBannedException +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config @@ -18,6 +21,7 @@ import io.github.manamiproject.modb.app.network.SuspendableHttpClient import io.github.manamiproject.modb.app.network.isRouteFailure import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.DefaultConfigRegistry +import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.core.coverage.KoverIgnore @@ -33,6 +37,7 @@ import kotlinx.coroutines.delay import java.net.ConnectException import java.net.NoRouteToHostException import java.net.UnknownHostException +import kotlin.io.path.deleteIfExists import kotlin.time.DurationUnit.MILLISECONDS import kotlin.time.DurationUnit.SECONDS import kotlin.time.toDuration @@ -42,6 +47,9 @@ import kotlin.time.toDuration * Uses [IntegerBasedIdRangeSelector] to determine which data to download. * Waits a random time between downloads and can move to another connection every few pages, both to reduce pressure * on the metadata provider. + * + * Reads `modb.app.crawler.deadEntriesToRecheck` like the other crawlers, but applies it to the pending entries. The + * AniDB dead entries come from the title dump, so a dead id doesn't exist and asking for it again is pointless. * @since 1.0.0 * @property appConfig Application specific configuration. Uses [AppConfig] by default. * @property metaDataProviderConfig Configuration for a specific metadata provider. @@ -54,6 +62,9 @@ import kotlin.time.toDuration * @property minWaitInMilliseconds Shortest wait before each download. * @property maxWaitInMilliseconds Longest wait before each download. * @property pagesPerConnection Downloads made through one connection before moving to the next. `0` stays on it. + * @property pendingEntries Entries whose addition is most likely still pending, in both modes. + * @property restrictedEntries 18+ entries, skipped in website mode. + * @property titleDump Source of the ids named by the title dump. */ class AnidbCrawler( private val appConfig: Config = AppConfig.instance, @@ -62,6 +73,7 @@ class AnidbCrawler( private val idRangeSelector: IdRangeSelector = IntegerBasedIdRangeSelector( metaDataProviderConfig = metaDataProviderConfig, highestIdDetector = AnidbTitlesDumpHighestIdDetector.instance, + recheckDeadEntries = false, ), private val httpClient: HttpClient = AnidbSource.httpClient(), private val downloader: Downloader = FaultTolerantDownloader( @@ -69,12 +81,25 @@ class AnidbCrawler( hostname = metaDataProviderConfig.hostname(), // Detection is answered below by leaving through another connection. Skipping it here // would burn the run's remaining entries against an address anidb has already refused. - isHandledByCaller = { it is CrawlerDetectedException }, + // A ban which outlasts the downloader's own retries stops the crawler right away for the + // same reason: every further request during a ban can extend it. + isHandledByCaller = { it is CrawlerDetectedException || it is AnidbBannedException }, ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), + private val pendingEntries: AnidbPendingEntries = DefaultAnidbPendingEntries.instance, + private val restrictedEntries: AnidbRestrictedEntries = DefaultAnidbRestrictedEntries.instance, + private val titleDump: suspend () -> Set = { AnidbTitlesDumpHighestIdDetector.instance.allIds() }, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ): Crawler { + private val deadEntriesToRecheck: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = IntegerBasedIdRangeSelector.CONFIG_NAMESPACE, + default = 0, + ) + + private var idsInTitleDump: Set? = null + private val maxConnectionChanges: Int by IntPropertyDelegate( configRegistry = configRegistry, namespace = CONFIG_NAMESPACE, @@ -125,9 +150,9 @@ class AnidbCrawler( val idDownloadList = idRangeSelector.idDownloadList().toMutableList() if (idDownloadList.isNotEmpty()) { - appConfig.workingDir(metaDataProviderConfig).listRegularFiles("*.${ANIDB_PENDING_FILE_SUFFIX}").forEach { - idDownloadList.remove(it.fileName().remove(".${ANIDB_PENDING_FILE_SUFFIX}").toInt()) - } + movePendingFilesToPendingEntries() + skipRestrictedEntries(idDownloadList) + skipPendingEntries(idDownloadList) when { idDownloadList.isEmpty() -> log.info { "No IDs left for [${metaDataProviderConfig.hostname()}] crawler to download." } @@ -138,6 +163,65 @@ class AnidbCrawler( log.info { "Finished crawling data for [${metaDataProviderConfig.hostname()}]." } } + /** + * Earlier versions kept a pending website entry as a `.pending` file in the week's working directory. + */ + private suspend fun movePendingFilesToPendingEntries() { + appConfig.workingDir(metaDataProviderConfig).listRegularFiles("*.${ANIDB_PENDING_FILE_SUFFIX}").forEach { file -> + file.fileName().remove(".${ANIDB_PENDING_FILE_SUFFIX}").toIntOrNull()?.let { pendingEntries.add(it) } + file.deleteIfExists() + } + } + + private suspend fun skipRestrictedEntries(idDownloadList: MutableList) { + if (!isWebsite() || restrictedEntries.all().isEmpty()) { + return + } + + restrictedEntries.removeAllNotIn(titleDumpIds()) + + val restricted = restrictedEntries.all() + idDownloadList.removeAll(restricted) + + log.info { "Skipping [${restricted.size}] 18+ entries of [${metaDataProviderConfig.hostname()}]." } + } + + private fun isWebsite(): Boolean = metaDataProviderConfig.fileSuffix() == AnidbWebsiteConfig.fileSuffix() + + private suspend fun skipPendingEntries(idDownloadList: MutableList) { + val pending = pendingEntries.all() + + if (pending.isEmpty()) { + return + } + + pendingEntries.removeAllNotIn(titleDumpIds()) + + val stillPending = pendingEntries.all() + val recheck = pendingEntries.selectForRecheck(deadEntriesToRecheck) + idDownloadList.removeAll(stillPending - recheck) + + log.info { "Skipping [${stillPending.size - recheck.size}] pending entries of [${metaDataProviderConfig.hostname()}], rechecking [${recheck.size}]." } + } + + private suspend fun titleDumpIds(): Set = idsInTitleDump ?: titleDump().also { idsInTitleDump = it } + + /** + * An id which the API refuses, but the title dump names, is not dead. Its addition is most likely still pending. + * The website only reports an entry as not served when the page says it has been removed from anidb. That is + * final, so it always becomes a dead entry. (Pending and 18+ entries are answered with an indicator instead.) + */ + private suspend fun handleNotServed(animeId: AnimeId) { + val id = animeId.toIntOrNull() + + if (!isWebsite() && id != null && id in titleDumpIds()) { + pendingEntries.add(id) + } else { + id?.let { pendingEntries.remove(it) } + deadEntriesAccess.addDeadEntry(animeId, metaDataProviderConfig) + } + } + private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> val animeId = idDownloadList[index] wait() @@ -153,14 +237,17 @@ class AnidbCrawler( val response = downloadChangingConnectionWhenRefused(animeId) when { - response.neitherNullNorBlank() && response != ANIDB_PENDING_FILE_INDICATOR -> { + response == ANIDB_PENDING_FILE_INDICATOR -> pendingEntries.add(animeId) + response == ANIDB_RESTRICTED_INDICATOR -> { + pendingEntries.remove(animeId) + restrictedEntries.add(animeId) + } + response.neitherNullNorBlank() -> { val file = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${metaDataProviderConfig.fileSuffix()}") response.writeToFile(file, true) deadEntriesAccess.removeDeadEntry(animeId.toString(), metaDataProviderConfig) - } - response == ANIDB_PENDING_FILE_INDICATOR -> { - val pendingFile = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${ANIDB_PENDING_FILE_SUFFIX}") - response.writeToFile(pendingFile, true) + pendingEntries.remove(animeId) + restrictedEntries.remove(animeId) } } } @@ -179,7 +266,7 @@ class AnidbCrawler( while (true) { try { return downloader.download(animeId.toString()) { - deadEntriesAccess.addDeadEntry(it, metaDataProviderConfig) + handleNotServed(it) } } catch (e: Throwable) { if (e !is CrawlerDetectedException || connectionChanges >= maxConnectionChanges) { @@ -208,8 +295,8 @@ class AnidbCrawler( private const val DEFAULT_MAX_CONNECTION_CHANGES = 5 private const val DEFAULT_ROUTE_FAILURE_WAIT_IN_SECONDS = 5 - private const val DEFAULT_MIN_WAIT_IN_MILLISECONDS = 1500 - private const val DEFAULT_MAX_WAIT_IN_MILLISECONDS = 2500 + private const val DEFAULT_MIN_WAIT_IN_MILLISECONDS = 2000 + private const val DEFAULT_MAX_WAIT_IN_MILLISECONDS = 3000 private const val DEFAULT_PAGES_PER_CONNECTION = 0 /** @@ -225,7 +312,7 @@ class AnidbCrawler( val instance: AnidbCrawler by lazy { AnidbCrawler() } /** - * Suffix for pending files which have been downloaded, but cannot be converted. + * Suffix of the pending files which earlier versions wrote. They are moved to the pending entries on start. * @since 7.0.0 */ const val ANIDB_PENDING_FILE_SUFFIX: String = "pending" diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt new file mode 100644 index 00000000..5a373fd0 --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt @@ -0,0 +1,185 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS +import io.github.manamiproject.modb.core.extensions.regularFileExists +import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.time.LocalDate +import kotlin.io.path.readLines +import kotlin.io.path.writeText + +/** + * AniDB entries whose addition is still pending approval. The website says so, the API only refuses them, while the + * title dump already names them. + * + * Such an id is neither dead nor downloadable. As a dead entry it would leave the dead entries list with every new + * title dump and be requested again on every run, only to be refused again. Kept apart, it is skipped until a recheck + * is asked for, and it leaves the list on its own once it is downloaded or once the title dump no longer names it. + * @since 1.0.0 + */ +interface AnidbPendingEntries { + + /** + * Removes ids which the title dump no longer names. Those are dead entries now. + * @since 1.0.0 + * @param idsInTitleDump All ids named by the current title dump. + */ + suspend fun removeAllNotIn(idsInTitleDump: Set) + + /** + * @since 1.0.0 + * @return All pending ids. + */ + suspend fun all(): Set + + /** + * Selects the pending ids which have been checked longest ago and records today as their last check. + * @since 1.0.0 + * @param numberOfEntries Maximum number of ids to select. + * @return Ids to request again. + */ + suspend fun selectForRecheck(numberOfEntries: Int): Set + + /** + * Adds an id or, if it is pending already, records today as its last check. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun add(id: Int) + + /** + * Removes an id, because it has been downloaded. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun remove(id: Int) +} + +/** + * Keeps the pending entries in the download control state directory, one `id last-check` pair per line. + * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. + */ +class DefaultAnidbPendingEntries( + private val appConfig: Config = AppConfig.instance, +): AnidbPendingEntries { + + private val access = Mutex() + private var entries: MutableMap? = null + + override suspend fun removeAllNotIn(idsInTitleDump: Set) = update { current -> + val removed = current.keys.filterNot { it in idsInTitleDump } + removed.forEach { current.remove(it) } + + if (removed.isNotEmpty()) { + log.info { "Removed [${removed.size}] pending [anidb.net] entries which the title dump no longer names." } + } + } + + override suspend fun all(): Set = access.withLock { load().keys.toSet() } + + override suspend fun selectForRecheck(numberOfEntries: Int): Set { + if (numberOfEntries <= 0) { + return emptySet() + } + + var selected = emptySet() + + update { current -> + selected = current.entries + .sortedWith(compareBy> { it.value }.thenBy { it.key }) + .take(numberOfEntries) + .map { it.key } + .toSet() + + val today = LocalDate.now(appConfig.clock()) + selected.forEach { current[it] = today } + } + + return selected + } + + override suspend fun add(id: Int) = update { current -> + if (id !in current) { + log.info { "Keeping [anidbId=$id] as pending entry, its addition is most likely not approved yet." } + } + + current[id] = LocalDate.now(appConfig.clock()) + } + + override suspend fun remove(id: Int) = update { current -> + if (current.remove(id) != null) { + log.info { "[anidb.net] serves the pending entry [anidbId=$id] now." } + } + } + + private suspend fun update(change: (MutableMap) -> Unit) = access.withLock { + val current = load() + val before = current.toMap() + change(current) + + if (current != before) { + save(current) + } + } + + private suspend fun load(): MutableMap = withContext(LIMITED_FS) { + entries?.let { return@withContext it } + + val file = file() + val loaded = mutableMapOf() + + if (file.regularFileExists()) { + file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith(COMMENT) } + .forEach { line -> + val parts = line.split(' ') + val id = parts[0].toIntOrNull() ?: return@forEach + val lastCheck = parts.getOrNull(1)?.let { runCatching { LocalDate.parse(it) }.getOrNull() } ?: LocalDate.MIN + loaded[id] = lastCheck + } + } + + entries = loaded + return@withContext loaded + } + + private suspend fun save(current: Map) = withContext(LIMITED_FS) { + val file = file() + val content = buildString { + appendLine("$COMMENT AniDB ids whose addition is still pending. Format: id last-check") + current.toSortedMap().forEach { (id, lastCheck) -> appendLine("$id $lastCheck") } + } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeText(content) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + } + + private fun file() = appConfig.downloadControlStateDirectory().resolve(FILE_NAME) + + companion object { + private val log by LoggerDelegate() + private const val COMMENT = "#" + + /** + * Name of the file in the download control state directory. + * @since 1.0.0 + */ + const val FILE_NAME: String = "anidb-pending-entries.txt" + + /** + * Singleton of [DefaultAnidbPendingEntries]. + * @since 1.0.0 + */ + val instance: DefaultAnidbPendingEntries by lazy { DefaultAnidbPendingEntries() } + } +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt new file mode 100644 index 00000000..fea9e070 --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt @@ -0,0 +1,144 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS +import io.github.manamiproject.modb.core.extensions.regularFileExists +import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import kotlin.io.path.readLines +import kotlin.io.path.writeText + +/** + * AniDB entries which are 18+ restricted. The website shows no data for them without an account, so in website mode + * they are skipped on every run. The API serves them, so they don't apply in API mode. + * @since 1.0.0 + */ +interface AnidbRestrictedEntries { + + /** + * Removes ids which the title dump no longer names. Those are dead entries now. + * @since 1.0.0 + * @param idsInTitleDump All ids named by the current title dump. + */ + suspend fun removeAllNotIn(idsInTitleDump: Set) + + /** + * @since 1.0.0 + * @return All restricted ids. + */ + suspend fun all(): Set + + /** + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun add(id: Int) + + /** + * Removes an id, because it has been downloaded. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun remove(id: Int) +} + +/** + * Keeps the restricted entries in the download control state directory, one id per line. + * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. + */ +class DefaultAnidbRestrictedEntries( + private val appConfig: Config = AppConfig.instance, +): AnidbRestrictedEntries { + + private val access = Mutex() + private var entries: MutableSet? = null + + override suspend fun removeAllNotIn(idsInTitleDump: Set) = update { current -> + val removed = current.filterNot { it in idsInTitleDump } + current.removeAll(removed.toSet()) + + if (removed.isNotEmpty()) { + log.info { "Removed [${removed.size}] 18+ [anidb.net] entries which the title dump no longer names." } + } + } + + override suspend fun all(): Set = access.withLock { load().toSet() } + + override suspend fun add(id: Int) = update { current -> + if (current.add(id)) { + log.info { "Keeping [anidbId=$id] as 18+ entry, it is skipped on the website from now on." } + } + } + + override suspend fun remove(id: Int) = update { current -> + if (current.remove(id)) { + log.info { "[anidb.net] serves the 18+ entry [anidbId=$id] now." } + } + } + + private suspend fun update(change: (MutableSet) -> Unit) = access.withLock { + val current = load() + val before = current.toSet() + change(current) + + if (current != before) { + save(current) + } + } + + private suspend fun load(): MutableSet = withContext(LIMITED_FS) { + entries?.let { return@withContext it } + + val file = file() + val loaded = mutableSetOf() + + if (file.regularFileExists()) { + file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith(COMMENT) } + .mapNotNull { it.toIntOrNull() } + .forEach { loaded.add(it) } + } + + entries = loaded + return@withContext loaded + } + + private suspend fun save(current: Set) = withContext(LIMITED_FS) { + val file = file() + val content = buildString { + appendLine("$COMMENT AniDB ids which are 18+ restricted and skipped in website mode.") + current.sorted().forEach { appendLine(it.toString()) } + } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeText(content) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + } + + private fun file() = appConfig.downloadControlStateDirectory().resolve(FILE_NAME) + + companion object { + private val log by LoggerDelegate() + private const val COMMENT = "#" + + /** + * Name of the file in the download control state directory. + * @since 1.0.0 + */ + const val FILE_NAME: String = "anidb-restricted-entries.txt" + + /** + * Singleton of [DefaultAnidbRestrictedEntries]. + * @since 1.0.0 + */ + val instance: DefaultAnidbRestrictedEntries by lazy { DefaultAnidbRestrictedEntries() } + } +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt index c23f1036..4039ee79 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt @@ -1,42 +1,77 @@ package io.github.manamiproject.modb.app.crawlers.anidb +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.crawlers.HighestIdDetector +import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor +import io.github.manamiproject.modb.app.dataset.DefaultDeadEntriesAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DefaultDownloadControlStateAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK +import io.github.manamiproject.modb.core.extensions.RegularFile +import io.github.manamiproject.modb.core.extensions.regularFileExists import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.io.ByteArrayInputStream import java.net.URI +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.time.Duration +import java.time.Instant +import java.time.format.DateTimeFormatter import java.util.zip.GZIPInputStream +import kotlin.coroutines.cancellation.CancellationException +import kotlin.io.path.getLastModifiedTime +import kotlin.io.path.readBytes +import kotlin.io.path.writeBytes /** - * Determines the highest anime id from the title dump which AniDB publishes. + * Determines the highest anime id from the title dump which AniDB publishes and derives the dead entries from it. * * The dump is a plain file rather than part of the website, so it is reachable without the * protection that anidb.net sits behind, and AniDB publishes it precisely so that clients do not * have to search or crawl for ids. It is regenerated daily and names every anime that exists, which - * also means ids that were never assigned are not requested at all. + * also means ids that were never assigned or have been deleted are not requested at all: every id + * from 1 to the highest id which the dump doesn't name becomes a dead entry. + * + * AniDB asks not to download the dump more than once per day. The last download is kept in the + * download control state directory and is only replaced once it is older than [maxAge]. * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. * @property httpClient To download the dump. * @property url Location of the dump. + * @property deadEntriesAccessor Access to dead entries files. + * @property downloadControlStateAccessor Access to DCS files. + * @property maxAge Age after which the kept dump is downloaded again. + * @property metaDataProviderConfig Config of the configured AniDB source, under which the dead entries are kept. */ class AnidbTitlesDumpHighestIdDetector( + private val appConfig: Config = AppConfig.instance, private val httpClient: HttpClient = DefaultHttpClient(), private val url: URI = URI(TITLES_DUMP_URL), + private val deadEntriesAccessor: DeadEntriesAccessor = DefaultDeadEntriesAccessor.instance, + private val downloadControlStateAccessor: DownloadControlStateAccessor = DefaultDownloadControlStateAccessor.instance, + private val maxAge: Duration = Duration.ofDays(1), + private val metaDataProviderConfig: MetaDataProviderConfig = AnidbSource.config(), ): HighestIdDetector { - override suspend fun detectHighestId(): Int = withContext(LIMITED_NETWORK) { - log.info { "Fetching the AniDB title dump to determine the highest id." } + private val dumpAccess = Mutex() - val body = httpClient.get(url.toURL()).bodyAsByteArray() - val ids = parse(body) + override suspend fun detectHighestId(): Int = withContext(LIMITED_NETWORK) { + val ids = allIds() + val highestId = ids.max() - check(ids.isNotEmpty()) { "Title dump did not contain any anime id." } + log.info { "Title dump names [${ids.size}] anime, the highest id being [$highestId]." } - log.info { "Title dump names [${ids.size}] anime, the highest id being [${ids.max()}]." } + updateDeadEntries(ids, highestId) - return@withContext ids.max() + return@withContext highestId } /** @@ -45,9 +80,72 @@ class AnidbTitlesDumpHighestIdDetector( * @return Ids of all anime which exist on AniDB. */ suspend fun allIds(): Set = withContext(LIMITED_NETWORK) { - return@withContext parse(httpClient.get(url.toURL()).bodyAsByteArray()) + val ids = parse(dump()) + check(ids.isNotEmpty()) { "Title dump did not contain any anime id." } + return@withContext ids + } + + private suspend fun dump(): ByteArray = dumpAccess.withLock { + val file = dumpFile() + val now = appConfig.clock().instant() + val lastDownload = if (file.regularFileExists()) file.getLastModifiedTime().toInstant() else Instant.EPOCH + + if (Duration.between(lastDownload, now) < maxAge) { + val nextDownload = lastDownload.plus(maxAge) + log.info { "Using the AniDB title dump from [${format(lastDownload)}]. Next download possible from [${format(nextDownload)}] (in [${formatDuration(Duration.between(now, nextDownload))}])." } + return@withLock file.readBytes() + } + + log.info { "Downloading the AniDB title dump from [$url]." } + + try { + val body = httpClient.get(url.toURL()).bodyAsByteArray() + + // A broken or empty download must not replace a working dump, because the dead + // entries are derived from it. + check(parse(body).isNotEmpty()) { "Title dump did not contain any anime id." } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeBytes(body) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + + return@withLock body + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + if (!file.regularFileExists()) { + throw e + } + + log.warn { "Unable to download the AniDB title dump [${e.message}]. Using the one from [${format(lastDownload)}] instead." } + return@withLock file.readBytes() + } + } + + private suspend fun updateDeadEntries(ids: Set, highestId: Int) { + // Each dead entry removes the corresponding DCS file. If the dump names fewer anime than + // the dataset already has, it is more likely to be broken than AniDB having deleted them. + val highestIdAlreadyInDataset = downloadControlStateAccessor.highestIdAlreadyInDataset(metaDataProviderConfig) + + if (highestId < highestIdAlreadyInDataset) { + log.warn { "Not updating the [${metaDataProviderConfig.hostname()}] dead entries, because the highest id of the title dump [$highestId] is smaller than the highest id already in the dataset [$highestIdAlreadyInDataset]." } + return + } + + val deadEntries = (1..highestId) + .filterNot { ids.contains(it) } + .map { it.toString() } + .toSet() + + deadEntriesAccessor.replaceDeadEntries(deadEntries, metaDataProviderConfig) } + private fun format(instant: Instant): String = TIMESTAMP_FORMAT.format(instant.atZone(appConfig.clock().zone)) + + private fun formatDuration(duration: Duration): String = "%dh %02dm".format(duration.toHours(), duration.toMinutesPart()) + + private fun dumpFile(): RegularFile = appConfig.downloadControlStateDirectory().resolve(DUMP_FILE_NAME) + private fun parse(body: ByteArray): Set { // The file is gzipped content rather than a compressed transfer, so the http client has no // reason to unwrap it. @@ -67,6 +165,7 @@ class AnidbTitlesDumpHighestIdDetector( private val log by LoggerDelegate() private const val COMMENT = "#" + private val TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm") private const val SEPARATOR = '|' private const val GZIP_MAGIC_FIRST: Byte = 0x1f private const val GZIP_MAGIC_SECOND: Byte = 0x8b.toByte() @@ -77,6 +176,12 @@ class AnidbTitlesDumpHighestIdDetector( */ const val TITLES_DUMP_URL = "https://anidb.net/api/anime-titles.dat.gz" + /** + * Name of the file in the download control state directory which keeps the last download of the dump. + * @since 1.0.0 + */ + const val DUMP_FILE_NAME = "anidb-anime-titles.dat.gz" + /** * Singleton of [AnidbTitlesDumpHighestIdDetector]. * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt index 93eb26f6..1a1ad75b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt @@ -2,7 +2,6 @@ package io.github.manamiproject.modb.app.crawlers.anilist import io.github.manamiproject.modb.anilist.AnilistConfig import io.github.manamiproject.modb.anilist.AnilistDownloader -import io.github.manamiproject.modb.anilist.AnilistHttpClient import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.crawlers.Crawler @@ -45,10 +44,8 @@ class AnilistCrawler( ), private val downloader: Downloader = FaultTolerantDownloader( downloader = AnilistDownloader( - httpClient = SuspendableHttpClient( - httpClient = AnilistHttpClient.instance, + httpClient = SuspendableHttpClient(), ), - ), hostname = metaDataProviderConfig.hostname(), ), ): Crawler { @@ -88,14 +85,14 @@ class AnilistCrawler( @KoverIgnore private suspend fun waitAfterDeadEntry() { excludeFromTestContext(metaDataProviderConfig) { - delay(random(1000, 1500).toDuration(MILLISECONDS)) + delay(random(670, 1500).toDuration(MILLISECONDS)) } } @KoverIgnore private suspend fun wait() { excludeFromTestContext(metaDataProviderConfig) { - delay(random(1000, 1500).toDuration(MILLISECONDS)) + delay(random(670, 1500).toDuration(MILLISECONDS)) } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt index ba500b0f..39ca216b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt @@ -2,7 +2,6 @@ package io.github.manamiproject.modb.app.crawlers.anilist import io.github.manamiproject.modb.anilist.AnilistConfig import io.github.manamiproject.modb.anilist.AnilistHeaderCreator -import io.github.manamiproject.modb.anilist.AnilistHttpClient import io.github.manamiproject.modb.app.crawlers.HighestIdDetector import io.github.manamiproject.modb.app.extensions.checkedBody import io.github.manamiproject.modb.app.network.SuspendableHttpClient @@ -22,7 +21,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate */ class AnilistHighestIdDetector( private val metaDataProviderConfig: MetaDataProviderConfig = AnilistConfig, - private val httpClient: HttpClient = SuspendableHttpClient(httpClient = AnilistHttpClient.instance), + private val httpClient: HttpClient = SuspendableHttpClient(), ): HighestIdDetector { override suspend fun detectHighestId(): Int { @@ -39,7 +38,6 @@ class AnilistHighestIdDetector( url = requestUrl.toURL(), headers = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = requestUrl.toURL(), ), requestBody = requestBody, ).checkedBody(this::class) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt index bc145472..c80ba68e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt @@ -93,6 +93,13 @@ class AnimenewsnetworkCrawler( pages = pages.dropWhile { it != lastPage }.drop(1) } + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [${pages.firstOrNull() ?: "-"}] with [${pages.size}] pages left." + } + wait() pages.forEach { page -> diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt index 620d5703..e7ffbc30 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt @@ -84,6 +84,13 @@ class AnimePlanetCrawler( val lastPage = lastPageDetector.detectHighestId() val entriesNotScheduledForCurrentWeek = downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig) + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [$firstPage] of [$lastPage]." + } + wait() for (currentPage in firstPage..lastPage) { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt index 8b1a3150..46f8056c 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.anisearch +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchDownloader import io.github.manamiproject.modb.app.config.AppConfig @@ -70,7 +71,7 @@ class AnisearchCrawler( metaDataProviderConfig = metaDataProviderConfig, httpClient = httpClient, ), - hostname = metaDataProviderConfig.hostname(), + hostname = metaDataProviderConfig.siteName(), ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), ): Crawler { @@ -83,31 +84,38 @@ class AnisearchCrawler( } override suspend fun start() { - log.info { "Starting crawler for [${metaDataProviderConfig.hostname()}]." } + log.info { "Starting crawler for [${metaDataProviderConfig.siteName()}]." } downloadEntriesScheduledForCurrentWeek() wait() downloadEntriesUsingPagination() - log.info { "Finished crawling data for [${metaDataProviderConfig.hostname()}]." } + log.info { "Finished crawling data for [${metaDataProviderConfig.siteName()}]." } } private suspend fun downloadEntriesScheduledForCurrentWeek() { - log.info { "Downloading [${metaDataProviderConfig.hostname()}] entries scheduled for the current week." } + log.info { "Downloading [${metaDataProviderConfig.siteName()}] entries scheduled for the current week." } val ids = downloadControlStateScheduler.findEntriesScheduledForCurrentWeek(metaDataProviderConfig) - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) startDownload(ids.toList().createShuffledList()) - log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries scheduled for the current week." } + log.info { "Finished downloading [${metaDataProviderConfig.siteName()}] entries scheduled for the current week." } } private suspend fun downloadEntriesUsingPagination() { - log.info { "Downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } + log.info { "Downloading [${metaDataProviderConfig.siteName()}] entries using pagination." } val firstPage = lastPageMemorizer.retrieveLastPage() val lastPage = lastPageDetector.detectHighestId() val entriesNotScheduledForCurrentWeek = downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig) + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.siteName()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [$firstPage] of [$lastPage]." + } + wait() for (currentPage in firstPage..lastPage) { @@ -117,7 +125,7 @@ class AnisearchCrawler( lastPageMemorizer.memorizeLastPage(currentPage) } - log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } + log.info { "Finished downloading [${metaDataProviderConfig.siteName()}] entries using pagination." } } private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt index 347828e1..ea3ab8ac 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.anisearch +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.convfiles.DefaultAlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.crawlers.PaginationIdRangeSelector @@ -58,7 +59,7 @@ class AnisearchPaginationIdRangeSelector( } override suspend fun idDownloadList(page: Int): List { - log.info { "Retrieving IDs for [${metaDataProviderConfig.hostname()}] from page [$page]" } + log.info { "Retrieving IDs for [${metaDataProviderConfig.siteName()}] from page [$page]" } if (entriesNotScheduledForCurrentWeek.isEmpty()) { entriesNotScheduledForCurrentWeek.addAll(downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig)) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt index 6ef49dd6..bd791484 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder @@ -27,6 +28,8 @@ import io.github.manamiproject.modb.core.random import io.github.manamiproject.modb.livechart.LivechartConfig import io.github.manamiproject.modb.livechart.LivechartDownloader import kotlinx.coroutines.delay +import java.time.Clock +import java.time.LocalDate import kotlin.time.DurationUnit.MILLISECONDS import kotlin.time.toDuration @@ -53,10 +56,11 @@ class LivechartCrawler( private val deadEntriesAccessor: DeadEntriesAccessor = DefaultDeadEntriesAccessor.instance, private val lastPageMemorizer: LastPageMemorizer = StringBasedLastPageMemorizer(metaDataProviderConfig = metaDataProviderConfig), private val newestYearDetector: HighestIdDetector = LivechartNewestYearDetector.instance, + private val clock: () -> Clock = { appConfig.clock() }, private val paginationIdRangeSelector: PaginationIdRangeSelector = LivechartPaginationIdRangeSelector.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, private val downloader: Downloader = FaultTolerantDownloader( - downloader = LivechartDownloader(httpClient = SuspendableHttpClient()), + downloader = LivechartDownloader(httpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase)), hostname = metaDataProviderConfig.hostname(), ), ): Crawler { @@ -92,15 +96,40 @@ class LivechartCrawler( pages = pages.dropWhile { it != lastPage }.drop(1) } + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [${pages.firstOrNull() ?: "-"}] with [${pages.size}] pages left." + } + wait() + val handledInThisRun = hashSetOf() + pages.forEach { page -> val currentList = paginationIdRangeSelector.idDownloadList(page) - entriesNotScheduledForCurrentWeek - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) wait() startDownload(currentList) + handledInThisRun.addAll(currentList) lastPageMemorizer.memorizeLastPage(page) } + // New anime appear in the current and the upcoming seasons, so these are looked at on every run, even once + // every page has been crawled this week. They aren't memorized, so they don't move where the crawl resumes. + val recheck = recheckPages(newestYear) - pages.toSet() + + if (recheck.isNotEmpty()) { + log.info { "[${metaDataProviderConfig.hostname()}] Rechecking [${recheck.size}] pages for new anime: $recheck" } + } + + recheck.forEach { page -> + val currentList = paginationIdRangeSelector.idDownloadList(page) - entriesNotScheduledForCurrentWeek - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) - handledInThisRun + wait() + startDownload(currentList) + handledInThisRun.addAll(currentList) + } + log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } } @@ -116,6 +145,19 @@ class LivechartCrawler( .sorted() } + /** + * Pages from the current season up to the last season of the newest year, followed by `tba`. + */ + private fun recheckPages(newestYear: Year): List { + val today = LocalDate.now(clock()) + val currentSeason = (today.monthValue - 1) / 3 + + return (today.year..newestYear).flatMap { year -> + SEASONS_IN_ORDER.filterIndexed { index, _ -> year > today.year || index >= currentSeason } + .map { season -> "$season-$year" } + } + "tba" + } + private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> val animeId = idDownloadList[index] val file = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${metaDataProviderConfig.fileSuffix()}") @@ -143,6 +185,11 @@ class LivechartCrawler( companion object { private val log by LoggerDelegate() + /** + * Seasons in the order of the year: January to March is winter, October to December is fall. + */ + private val SEASONS_IN_ORDER = listOf("winter", "spring", "summer", "fall") + /** * Singleton of [LivechartCrawler] * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt index ca89e3c1..871fb207 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.crawlers.HighestIdDetector import io.github.manamiproject.modb.app.extensions.checkedBody import io.github.manamiproject.modb.app.network.SuspendableHttpClient @@ -24,31 +25,38 @@ import kotlin.time.toDuration */ class LivechartNewestYearDetector( private val metaDataProviderConfig: MetaDataProviderConfig = LivechartNewestYearDetectorConfig, - private val httpClient: HttpClient = SuspendableHttpClient(), + private val httpClient: HttpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase), private val extractor: DataExtractor = XmlDataExtractor, ): HighestIdDetector { override suspend fun detectHighestId(): Int { - log.info { "Fetching highest id for [${metaDataProviderConfig.hostname()}]." } + log.info { "Detecting newest year for [${metaDataProviderConfig.hostname()}]." } wait() + val url = metaDataProviderConfig.buildDataDownloadLink().toURL() val response = httpClient.get( - url = metaDataProviderConfig.buildDataDownloadLink().toURL(), + url = url, headers = mapOf("host" to listOf("www.${metaDataProviderConfig.hostname()}")), ).checkedBody(this::class) + // The season links are read instead of the headings, because they survive a change of the layout. val data = extractor.extract(response, mapOf( - "headers" to "//ul[@class='grouped-list']/li//h5", + "links" to "//a/@href", + "title" to "//title/text()", )) - if (data.notFound("headers")) { - throw IllegalStateException("Unable to extract headers.") + val years = when { + data.notFound("links") -> emptyList() + else -> data.listNotNull("links").mapNotNull { SEASON_LINK.find(it.trim())?.groupValues?.get(1)?.toInt() } } - val headers = data.listNotNull("headers") + if (years.isEmpty()) { + val title = data.stringOrDefault("title").trim().ifEmpty { "none" } + throw IllegalStateException("No list of years found on [$url], page title [$title]. Most likely a block or maintenance page, or the layout has changed.") + } - return headers[1].trim().toInt() + return years.max() } @KoverIgnore @@ -61,6 +69,11 @@ class LivechartNewestYearDetector( companion object { private val log by LoggerDelegate() + /** + * A season page, such as `/winter-2027` or `/winter-2026/tv`, either relative or as full link. + */ + private val SEASON_LINK = Regex("""^(?:https?://(?:www\.)?livechart\.me)?/(?:winter|spring|summer|fall)-(\d{4})(?:/.*)?$""") + /** * Singleton of [LivechartNewestYearDetector] * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt index dd04ff7c..0067bd1e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.convfiles.DefaultAlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.crawlers.PaginationIdRangeSelector @@ -35,7 +36,7 @@ import kotlin.time.toDuration */ class LivechartPaginationIdRangeSelector( private val metaDataProviderConfig: MetaDataProviderConfig = LivechartPaginationIdRangeSelectorConfig, - private val httpClient: HttpClient = SuspendableHttpClient(), + private val httpClient: HttpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase), private val extractor: DataExtractor = XmlDataExtractor, private val downloadControlStateScheduler: DownloadControlStateScheduler = DefaultDownloadControlStateScheduler.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, @@ -57,12 +58,13 @@ class LivechartPaginationIdRangeSelector( headers = mapOf("host" to listOf("www.${metaDataProviderConfig.hostname()}")), ).use { response -> if (response.code == 404) { - log.warn { "404 on LivechartPaginationIdRangeSelector" } + log.warn { "[${metaDataProviderConfig.hostname()}] page [$page] doesn't exist (404). Treating it as empty." } return emptyList() } val data = extractor.extract(response.checkedBody(this::class), mapOf( - "animeIdList" to "//article[@class='anime']/@data-anime-id", + // Any card carrying an id: the class of the card changed with the redesign of october 2026. + "animeIdList" to "//article[@data-anime-id]/@data-anime-id", "emptyMessage" to "//section[@data-anime-card-list-target='emptyMessage']/text()", )) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt index eada26cd..98cd9c75 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt @@ -87,6 +87,12 @@ class SimklCrawler( } val maxYear = LocalDate.now(appConfig.clock()).year + 2 + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. Continuing from year [$startYear] up to [$maxYear]." + } + for (currentYear in startYear..maxYear) { for (currentOffset in 0..Int.MAX_VALUE step 20) { log.debug { "Checking year [$currentYear] with offset [$currentOffset] for [${metaDataProviderConfig.hostname()}]." } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt index 1e2488f3..a9a58437 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt @@ -38,6 +38,20 @@ interface DeadEntriesAccessor { */ suspend fun removeDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) + /** + * Replaces the whole dead entries list of a metadata provider. + * For metadata providers which publish a complete list of their anime. Ids which are new to the list are + * handled like [addDeadEntry], ids which are no longer part of it like [removeDeadEntry]. + * @since 1.0.0 + * @param animeIds Complete list of dead entries. + * @param metaDataProviderConfig Configuration for a specific metadata provider. + */ + suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + val current = fetchDeadEntries(metaDataProviderConfig) + (current - animeIds).forEach { removeDeadEntry(it, metaDataProviderConfig) } + (animeIds - current).forEach { addDeadEntry(it, metaDataProviderConfig) } + } + /** * Find dead entries in a list of [URI]. * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt index dab01280..c819fbca 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt @@ -121,6 +121,42 @@ class DefaultDeadEntriesAccessor( } } + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + if (!appConfig.deadEntriesSupported(metaDataProviderConfig)) { + return + } + + if (!isInitialized) { + init() + } + + val added = writeAccess.withLock { + val deadEntriesOfProvider = when (metaDataProviderConfig.hostname()) { + AnidbConfig.hostname() -> deadEntries.anidb + AnilistConfig.hostname() -> deadEntries.anilist + AnimenewsnetworkConfig.hostname() -> deadEntries.animenewsnetwork + KitsuConfig.hostname() -> deadEntries.kitsu + MyanimelistConfig.hostname() -> deadEntries.myanimelist + else -> throw IllegalStateException("Metadata provider [${metaDataProviderConfig.hostname()}] is not supported.") + } + + val added = animeIds - deadEntriesOfProvider + val removed = deadEntriesOfProvider - animeIds + + log.info { "[${metaDataProviderConfig.hostname()}] has [${animeIds.size}] dead entries. Adding [${added.size}] and removing [${removed.size}]." } + + if (added.isNotEmpty() || removed.isNotEmpty()) { + deadEntriesOfProvider.clear() + deadEntriesOfProvider.addAll(animeIds) + jsonSerializer.serialize(deadEntriesOfProvider, minify = true).writeToFile(deadEntriesFile(metaDataProviderConfig, JSON_MINIFIED)) + } + + added + } + + added.forEach { downloadControlStateAccessor.removeDeadEntry(metaDataProviderConfig, it) } + } + override suspend fun determineDeadEntries(sources: Collection): Set { if (!isInitialized) { init() diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt index bde6a2e4..d51425f5 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt @@ -139,8 +139,24 @@ class DefaultDownloadControlStateAccessor( val currentDownloadControlStateEntry = dcsEntry(metaDataProviderConfig, animeId) if (currentDownloadControlStateEntry.lastDownloaded == WeekOfYear.currentWeek()) { - log.debug { "Not updating DCS file for [${animeId}] of [${metaDataProviderConfig.hostname()}], because it has been updated already." } - return false + // A second run in the same week must not count as another week of the schedule. But if the + // raw files have been converted again in the meantime (e.g. by a fixed converter), the data + // of this week has changed and is kept, otherwise the dataset is built from the old version. + if (currentDownloadControlStateEntry.anime == downloadControlStateEntry.anime) { + log.debug { "Not updating DCS file for [${animeId}] of [${metaDataProviderConfig.hostname()}], because it has been updated already." } + return false + } + + log.debug { "Refreshing the data of DCS entry [$animeId] of [${metaDataProviderConfig.hostname()}], which has been updated this week already, without changing its schedule." } + + val refreshed = currentDownloadControlStateEntry.copy(_anime = downloadControlStateEntry.anime) + Json.toJson(refreshed).writeToFile(downloadControlStateFile) + safelyStore(metaDataProviderConfig) { + it.toMutableMap().apply { + put(animeId, refreshed) + } + } + return true } log.info { "Updating DCS entry for [$animeId] of [${metaDataProviderConfig.hostname()}]." } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt index 1cc092c9..c136fb2e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.downloadcontrolstate +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.core.config.AnimeId @@ -22,7 +23,7 @@ class DefaultDownloadControlStateScheduler( ): DownloadControlStateScheduler { override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are not scheduled to download this week." } + log.info { "Finding all anime which are not scheduled to download this week for [${metaDataProviderConfig.siteName()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isAfterCurrentWeek(it.nextDownload) } @@ -31,7 +32,7 @@ class DefaultDownloadControlStateScheduler( } override suspend fun findEntriesScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are scheduled to download this week." } + log.info { "Finding all anime which are scheduled to download this week for [${metaDataProviderConfig.siteName()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isCurrentWeekOrOlder(it.nextDownload) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt index a48aba5a..10139225 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt @@ -134,10 +134,26 @@ class DefaultDownloadControlStateUpdater( .map { workDir -> workDir.listRegularFiles("*.$CONVERTED_FILE_SUFFIX").map { file -> async { - Json.parseJson(file.readFile())!! to file.fileName() + val anime = try { + Json.parseJson(file.readFile()) + } catch (e: Exception) { + log.debug { "Unable to parse [$file]: ${e.message}" } + null + } + + when (anime) { + null -> { + // Most likely written only partially, because the app was stopped during the + // conversion. Without the file the raw file counts as unconverted again. + file.deleteIfExists() + log.warn { "Skipping [${file.fileName()}] of [${workDir.fileName()}], because it is empty or broken. It has been deleted and will be converted again on the next start." } + null + } + else -> anime to file.fileName() + } } } - }.flatten().awaitAll() + }.flatten().awaitAll().filterNotNull() return@withContext jobsResults } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt new file mode 100644 index 00000000..2df32b34 --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt @@ -0,0 +1,16 @@ +package io.github.manamiproject.modb.app.extensions + +import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig + +/** + * Name of the crawler to use in log messages. Equals the hostname, except for crawlers which share a hostname with + * another one. The names are the same as the ones used for `modb.app.disabledCrawlers`. + * @since 20.0.0 + * @receiver Config of the crawler. + * @return Name which identifies the crawler. + */ +fun MetaDataProviderConfig.siteName(): String = when (this) { + AnisearchRelationsConfig -> "${hostname()}-relations" + else -> hostname() +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt index e035937c..1bea7ace 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt @@ -67,6 +67,7 @@ class ClearanceHttpClient( renewedOnThisRoute = true if (clearance.refresh(url, route)) { + response.close() continue } @@ -80,6 +81,8 @@ class ClearanceHttpClient( return response } + // A refusal which isn't returned must be closed, otherwise its connection is never released. + response.close() changeConnection() renewedOnThisRoute = false } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt index 3821f538..27ae6b5b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt @@ -11,6 +11,7 @@ import io.github.manamiproject.modb.core.json.Json import io.github.manamiproject.modb.core.logging.LoggerDelegate import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.net.ConnectException import java.net.URI import java.net.URL import java.nio.file.Files @@ -125,7 +126,14 @@ class CloudflareClearance( body = """{"cmd":"request.get","url":"$url","maxTimeout":$SOLVER_TIMEOUT,"disableMedia":true$proxy}""", ) - val response = httpClient.post(URI(solverUrl).toURL(), body).bodyAsString() + val response = try { + httpClient.post(URI(solverUrl).toURL(), body).bodyAsString() + } catch (e: ConnectException) { + // OkHttp's message only names an address, which reads as if the site itself were unreachable. + // It stays a ConnectException, so the callers' handling doesn't change. + val address = e.message?.substringAfter("Failed to connect to ", missingDelimiterValue = "")?.ifBlank { null } ?: solverUrl + throw ConnectException("Failed to connect to the challenge solver [$address]").apply { initCause(e) } + } val parsed = Json.parseJson(response) if (parsed?.status != "ok" || parsed.solution == null) { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt index 562b0a7f..d26c4eac 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt @@ -124,7 +124,8 @@ class LinuxNetworkController( ifconfig(device, option) { commandExecutor = kotlinCommandExecutor.apply { - config.useSudo = true + // No password means the app runs as root and doesn't need sudo. + config.useSudo = sudoPasswordValue.isNotBlank() config.sudoPassword = sudoPasswordValue } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt index be04f271..a0c2399d 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt @@ -21,6 +21,7 @@ object NetworkControllers { private val rotatingControllers = mutableMapOf() private val routedControllers = mutableMapOf, RotatingProxyNetworkController>() + private val loggedDecisions = mutableSetOf>() /** * Returns the one rotating controller belonging to a configuration. @@ -100,16 +101,39 @@ object NetworkControllers { return when { rotating.hasProxies() -> rotating canRestartNetworkDevice() -> { - log.info { "Using the network device to change the outgoing address." } + logOncePerSite(hostname, "Using the network device to change the outgoing address.") LinuxNetworkController.instance } else -> { - log.info { "No proxy pool configured and no network device to restart, so the outgoing address cannot be changed on demand." } + logOncePerSite(hostname, "No proxy pool configured and no network device to restart, so the outgoing address cannot be changed on demand.") FixedNetworkController } } } + // Several classes of the same crawler ask for a controller, which would repeat the same message for each of them. + private fun logOncePerSite(hostname: String?, message: String) { + val site = hostname ?: "all sites" + val isNew = synchronized(loggedDecisions) { loggedDecisions.add(site to message) } + + if (isNew) { + log.info { "[$site] $message" } + } + } + + /** + * Tells in advance whether [forDeployment] will hand out [LinuxNetworkController] for any of the given sites, + * which is the only case in which a sudo password is needed. + * @since 20.0.0 + * @param hostnames Sites whose crawlers change the outgoing address on demand. + * @param configRegistry Source of the proxy pool definition. + * @return `true` if at least one of the sites has no proxies and the network device can be restarted. + */ + fun usesNetworkDevice(hostnames: Collection, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance): Boolean { + val withoutProxies = hostnames.filterNot { forHost(it, configRegistry).hasProxies() } + return withoutProxies.isNotEmpty() && canRestartNetworkDevice() + } + // The original design changes address by restarting the device, which only works where ifconfig // exists and the connection hands out a new address afterwards. That is a property of the // machine, so ask it rather than assume either way. diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt index 3c73609c..9b717e6b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt @@ -32,12 +32,12 @@ class SuspendableHttpClient( ): HttpClient { override suspend fun get(url: URL, headers: Map>): HttpResponse { - suspendIfNecessary() + suspendIfNecessary(url) return httpClient.get(url, headers) } override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { - suspendIfNecessary() + suspendIfNecessary(url) return httpClient.post(url, requestBody, headers) } @@ -46,9 +46,9 @@ class SuspendableHttpClient( return this } - private suspend fun suspendIfNecessary() = withContext(Default) { + private suspend fun suspendIfNecessary(url: URL) = withContext(Default) { while(!networkController.isNetworkActive() && isActive) { - log.info { "Waiting for network to be active again." } + log.info { "Waiting for network to be active again before requesting [${url.host}]." } wait() } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt index e4ecebad..bfbb53db 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt @@ -59,7 +59,8 @@ internal fun ifconfig(device: Device, vararg options: String, config: IfconfigCo val currentConfig = IfconfigConfig().apply(config) val commandBuilder = mutableListOf() - if (options.contains("up") || options.contains("down")) { + // Only with sudo enabled. Running as root, the executor rejects a command starting with sudo. + if ((options.contains("up") || options.contains("down")) && currentConfig.commandExecutor.config.useSudo) { commandBuilder.add("sudo") } commandBuilder.add("ifconfig") diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt index 312573ff..97d977a2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt @@ -159,10 +159,26 @@ internal object TestDeadEntriesAccessor: DeadEntriesAccessor { override fun deadEntriesFile(metaDataProviderConfig: MetaDataProviderConfig, type: DatasetFileType): RegularFile = shouldNotBeInvoked() override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() override suspend fun removeDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { } + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() override suspend fun determineDeadEntries(sources: Collection): Set = shouldNotBeInvoked() override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = shouldNotBeInvoked() } +internal object TestAnidbPendingEntries: io.github.manamiproject.modb.app.crawlers.anidb.AnidbPendingEntries { + override suspend fun removeAllNotIn(idsInTitleDump: Set) { } + override suspend fun all(): Set = emptySet() + override suspend fun selectForRecheck(numberOfEntries: Int): Set = emptySet() + override suspend fun add(id: Int) { } + override suspend fun remove(id: Int) { } +} + +internal object TestAnidbRestrictedEntries: io.github.manamiproject.modb.app.crawlers.anidb.AnidbRestrictedEntries { + override suspend fun removeAllNotIn(idsInTitleDump: Set) { } + override suspend fun all(): Set = emptySet() + override suspend fun add(id: Int) { } + override suspend fun remove(id: Int) { } +} + internal object TestReviewedIsolatedEntriesAccessor: ReviewedIsolatedEntriesAccessor { override fun contains(uri: URI): Boolean = shouldNotBeInvoked() override suspend fun addCheckedEntry(uri: URI) = shouldNotBeInvoked() diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt index cddaf5ff..68019a90 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt @@ -24,6 +24,9 @@ import org.junit.jupiter.api.Nested import java.nio.file.Path import kotlin.io.path.createDirectory import kotlin.io.path.createFile +import kotlin.io.path.fileSize +import kotlin.io.path.readText +import kotlin.io.path.writeText import kotlin.io.path.deleteIfExists import kotlin.io.path.listDirectoryEntries import kotlin.test.Test @@ -78,6 +81,50 @@ internal class SimpleConversionWatchServiceTest { } } + @Test + fun `deletes empty conv files when starting the watch service and converts their raw files again`() { + tempDirectory { + // given + val anime = AnimeRaw("Death Note") + + val testAppConfig = object: Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + override fun fileSuffix(): FileSuffix = "html" + } + + val convertedPaths = mutableListOf() + val testConverter = object: PathAnimeConverter by TestPathAnimeConverter { + override suspend fun convert(path: Path): List { + convertedPaths.add(path) + return listOf(anime) + } + } + + tempDir.resolve("1535.${testConfig.fileSuffix()}").createFile() + val emptyConvFile = tempDir.resolve("1535.$CONVERTED_FILE_SUFFIX").createFile() + tempDir.resolve("4000.${testConfig.fileSuffix()}").createFile() + tempDir.resolve("4000.$CONVERTED_FILE_SUFFIX").writeText("{}") + + val simpleConversionWatchService = SimpleConversionWatchService( + appConfig = testAppConfig, + converter = testConverter, + metaDataProviderConfig = testConfig, + ) + + // when + simpleConversionWatchService.prepare() + + // then + assertThat(convertedPaths.map { it.fileName() }).containsExactly("1535.${testConfig.fileSuffix()}") + assertThat(emptyConvFile.fileSize()).isGreaterThan(0L) + assertThat(tempDir.resolve("4000.$CONVERTED_FILE_SUFFIX").readText()).isEqualTo("{}") + } + } + @Test fun `convert all unconverted files at startup`() { tempDirectory { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt index 21999b4d..17f25ebc 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt @@ -6,6 +6,7 @@ import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateScheduler import io.github.manamiproject.modb.core.config.AnimeId +import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.Hostname import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.test.exceptionExpected @@ -246,6 +247,107 @@ internal class IntegerBasedIdRangeSelectorTest { } } + @Test + fun `rechecks the highest dead entries as configured by deadEntriesToRecheck`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + } + + val testHighestIdDetector = object: HighestIdDetector { + override suspend fun detectHighestId(): Int = 8 + } + + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = setOf("3", "5") + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = 5 + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testDefaultAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.app.crawler.deadEntriesToRecheck") 1 else null + } + + val idRangeSelector = IntegerBasedIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + highestIdDetector = testHighestIdDetector, + deadEntriesAccessor = testDeadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testDefaultAlreadyDownloadedIdsFinder, + configRegistry = testConfigRegistry, + ) + + // when + val result = idRangeSelector.idDownloadList() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 2, 4, 5, 6, 7, 8) + } + } + + @Test + fun `doesn't recheck dead entries if the crawler turned it off`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + } + + val testHighestIdDetector = object: HighestIdDetector { + override suspend fun detectHighestId(): Int = 8 + } + + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = setOf("3", "5") + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = 5 + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testDefaultAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.app.crawler.deadEntriesToRecheck") 1 else null + } + + val idRangeSelector = IntegerBasedIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + highestIdDetector = testHighestIdDetector, + deadEntriesAccessor = testDeadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testDefaultAlreadyDownloadedIdsFinder, + recheckDeadEntries = false, + configRegistry = testConfigRegistry, + ) + + // when + val result = idRangeSelector.idDownloadList() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 2, 4, 6, 7, 8) + } + } + @Test fun `removes IDs which are not scheduled for current week from generated sequence`() { runTest { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt index 03df50b3..628e7f5b 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt @@ -1,7 +1,9 @@ package io.github.manamiproject.modb.app.crawlers.anidb import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.* import io.github.manamiproject.modb.app.config.Config @@ -72,6 +74,9 @@ internal class AnidbCrawlerTest { httpClient = testHttpClient, downloader = TestDownloader, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // then @@ -117,6 +122,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -155,6 +163,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -169,7 +180,7 @@ internal class AnidbCrawlerTest { } @Test - fun `removes IDs of anime which are pending addition and have been downloaded already`() { + fun `moves pending files of earlier versions to the pending entries and skips them`() { tempDirectory { // given val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { @@ -196,6 +207,14 @@ internal class AnidbCrawlerTest { ANIDB_PENDING_FILE_INDICATOR.writeToFile(tempDir.resolve("1254.${ANIDB_PENDING_FILE_SUFFIX}")) + val pending = mutableSetOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + override suspend fun all(): Set = pending.toSet() + } + val crawler = AnidbCrawler( appConfig = testAppConfig, metaDataProviderConfig = testMetaDataProviderConfig, @@ -203,16 +222,21 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { setOf(1254) }, ) // when crawler.start() // then + assertThat(pending).containsExactly(1254) assertThat(tempDir).isDirectoryContaining { it.fileName() == "1535.xml" } assertThat(tempDir).isDirectoryContaining { it.fileName() == "23.xml" } assertThat(tempDir).isDirectoryContaining { it.fileName() == "424.xml" } assertThat(tempDir).isDirectoryNotContaining { it.fileName() == "1254.xml" } + assertThat(tempDir).isDirectoryNotContaining { it.fileName() == "1254.${ANIDB_PENDING_FILE_SUFFIX}" } } } @@ -253,6 +277,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = testDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -266,6 +293,194 @@ internal class AnidbCrawlerTest { } } + @Test + fun `keeps an id which the title dump names as pending entry instead of adding it to the dead entries`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1000, 1001) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + onDeadEntry.invoke(id) + return EMPTY + } + } + + val deadEntries = mutableListOf() + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { + deadEntries.add(animeId) + } + } + + val pending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = testDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { setOf(1000) }, + ) + + // when + crawler.start() + + // then + assertThat(pending).containsExactly(1000) + assertThat(deadEntries).containsExactly("1001") + } + } + + @Test + fun `on the website, an entry reported as removed is dead even though the title dump names it`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1000, 1001) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + onDeadEntry.invoke(id) + return EMPTY + } + } + + val deadEntries = mutableListOf() + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { + deadEntries.add(animeId) + } + } + + val added = mutableListOf() + val removed = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + added.add(id) + } + override suspend fun remove(id: Int) { + removed.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = testDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { setOf(1000) }, + ) + + // when + crawler.start() + + // then + assertThat(added).isEmpty() + assertThat(removed).containsExactly(1000, 1001) + assertThat(deadEntries).containsExactly("1000", "1001") + } + } + + @Test + fun `skips pending entries and only requests those selected for a recheck`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1, 2, 3, 4) + } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } + } + + var recheckRequestedFor = -1 + val removed = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun all(): Set = setOf(2, 3) + override suspend fun selectForRecheck(numberOfEntries: Int): Set { + recheckRequestedFor = numberOfEntries + return setOf(3) + } + override suspend fun remove(id: Int) { + removed.add(id) + } + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = when (key) { + "modb.app.crawler.deadEntriesToRecheck" -> 1 + else -> null + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { setOf(1, 2, 3, 4) }, + configRegistry = testConfigRegistry, + ) + + // when + crawler.start() + + // then + assertThat(recheckRequestedFor).isEqualTo(1) + assertThat(requested).containsExactlyInAnyOrder("1", "3", "4") + assertThat(removed).contains(3) + } + } + @Test fun `directly throws an exception if it's not one of the cases that restart the network controller`() { tempDirectory { @@ -295,6 +510,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -334,6 +552,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -345,7 +566,7 @@ internal class AnidbCrawlerTest { } @Test - fun `creates a file which indicates that the respective anime is pending addition and won't require another download`() { + fun `adds an entry whose addition is pending to the pending entries instead of creating a file`() { tempDirectory { // given val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { @@ -364,6 +585,122 @@ internal class AnidbCrawlerTest { override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = ANIDB_PENDING_FILE_INDICATOR } + val pending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, + ) + + // when + crawler.start() + + // then + assertThat(pending).containsExactly(1000) + assertThat(tempDir.listDirectoryEntries()).isEmpty() + } + } + + @Test + fun `on the website, keeps an 18+ entry in the restricted entries`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = ANIDB_RESTRICTED_INDICATOR + } + + val removedFromPending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun remove(id: Int) { + removedFromPending.add(id) + } + } + + val restricted = mutableListOf() + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun add(id: Int) { + restricted.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531) }, + ) + + // when + crawler.start() + + // then + assertThat(restricted).containsExactly(3531) + assertThat(removedFromPending).containsExactly(3531) + assertThat(tempDir.listDirectoryEntries()).isEmpty() + } + } + + @Test + fun `on the website, skips the restricted entries which the title dump still names`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531, 23) + } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } + } + + val restricted = mutableSetOf(3531, 999) + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun all(): Set = restricted.toSet() + override suspend fun removeAllNotIn(idsInTitleDump: Set) { + restricted.retainAll(idsInTitleDump) + } + } + val crawler = AnidbCrawler( appConfig = testAppConfig, metaDataProviderConfig = testMetaDataProviderConfig, @@ -371,15 +708,70 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531, 23) }, ) // when crawler.start() // then - assertThat(tempDir).isDirectoryContaining { file -> - file.fileName() == "1000.pending" + assertThat(requested).containsExactly("23") + assertThat(restricted).containsExactly(3531) + } + } + + @Test + fun `in API mode, requests restricted entries like any other`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531) + } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } } + + val removedFromRestricted = mutableListOf() + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun all(): Set = setOf(3531) + override suspend fun remove(id: Int) { + removedFromRestricted.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531) }, + ) + + // when + crawler.start() + + // then + assertThat(requested).containsExactly("3531") + assertThat(removedFromRestricted).containsExactly(3531) } } @@ -425,6 +817,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -468,6 +863,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -529,6 +927,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = testDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, ) // when @@ -580,6 +981,9 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { emptySet() }, configRegistry = testConfigRegistry, ) } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt index ae739b45..e3e2f319 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt @@ -1,28 +1,59 @@ package io.github.manamiproject.modb.app.crawlers.anidb +import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.TestDeadEntriesAccessor +import io.github.manamiproject.modb.app.TestDownloadControlStateAccessor +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.AnidbTitlesDumpHighestIdDetector.Companion.DUMP_FILE_NAME +import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor +import io.github.manamiproject.modb.core.config.AnimeId +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.extensions.Directory import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.httpclient.HttpResponse import io.github.manamiproject.modb.core.httpclient.RequestBody import io.github.manamiproject.modb.core.httpclient.RetryCase import io.github.manamiproject.modb.test.exceptionExpected import io.github.manamiproject.modb.test.shouldNotBeInvoked +import io.github.manamiproject.modb.test.tempDirectory import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested +import java.io.IOException import java.net.URL +import java.nio.file.attribute.FileTime +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.io.path.readBytes +import kotlin.io.path.setLastModifiedTime +import kotlin.io.path.writeBytes import kotlin.test.Test internal class AnidbTitlesDumpHighestIdDetectorTest { + private val now = Instant.parse("2026-10-29T12:00:00Z") + private val smallDump = """ + # created: whenever + 1|1|x-jat|First + 3|1|x-jat|Third + 3|4|en|Third in english + 5|1|x-jat|Fifth + """.trimIndent().toByteArray() + @Nested inner class DetectHighestIdTests { @Test fun `reads the highest id from the gzipped dump`() { - runBlocking { + tempDirectory { // given val dump = this::class.java.classLoader.getResourceAsStream("crawlers/anidb/anime-titles.dat.gz")!!.readBytes() - val detector = AnidbTitlesDumpHighestIdDetector(httpClient = clientReturning(dump)) + val detector = detector(tempDir, clientReturning(dump)) // when val result = detector.detectHighestId() @@ -34,10 +65,10 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { @Test fun `ignores comments and reads every id`() { - runBlocking { + tempDirectory { // given val dump = this::class.java.classLoader.getResourceAsStream("crawlers/anidb/anime-titles.dat.gz")!!.readBytes() - val detector = AnidbTitlesDumpHighestIdDetector(httpClient = clientReturning(dump)) + val detector = detector(tempDir, clientReturning(dump)) // when val result = detector.allIds() @@ -51,21 +82,234 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { @Test fun `throws if the dump contains no id`() { - // given - val detector = AnidbTitlesDumpHighestIdDetector( - httpClient = clientReturning("# created: whenever\n".toByteArray()), - ) + tempDirectory { + // given + val detector = detector(tempDir, clientReturning("# created: whenever\n".toByteArray())) + + // when + val result = exceptionExpected { + detector.detectHighestId() + } + + // then + assertThat(result).hasMessage("Title dump did not contain any anime id.") + } + } + + @Test + fun `every id up to the highest id which the dump doesn't name becomes a dead entry`() { + tempDirectory { + // given + var replacedWith = emptySet() + var replacedFor: MetaDataProviderConfig? = null + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + replacedWith = animeIds + replacedFor = metaDataProviderConfig + } + } + + val detector = detector(tempDir, clientReturning(smallDump), deadEntriesAccessor = testDeadEntriesAccessor) + + // when + val result = detector.detectHighestId() - // when - val result = exceptionExpected { + // then + assertThat(result).isEqualTo(5) + assertThat(replacedWith).containsExactlyInAnyOrder("2", "4") + assertThat(replacedFor).isEqualTo(AnidbConfig) + } + } + + @Test + fun `keeps the dead entries under the website config when the website is the source`() { + tempDirectory { + // given + var replacedWith = emptySet() + var replacedFor: MetaDataProviderConfig? = null + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + replacedWith = animeIds + replacedFor = metaDataProviderConfig + } + } + + val detector = detector( + tempDir = tempDir, + httpClient = clientReturning(smallDump), + deadEntriesAccessor = testDeadEntriesAccessor, + metaDataProviderConfig = AnidbWebsiteConfig, + ) + + // when detector.detectHighestId() + + // then + assertThat(replacedWith).containsExactlyInAnyOrder("2", "4") + assertThat(replacedFor).isEqualTo(AnidbWebsiteConfig) } + } + + @Test + fun `doesn't touch the dead entries if the dump ends below the highest id already in the dataset`() { + tempDirectory { + // given + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() + } + + val detector = detector( + tempDir = tempDir, + httpClient = clientReturning(smallDump), + deadEntriesAccessor = testDeadEntriesAccessor, + highestIdAlreadyInDataset = 6, + ) + + // when + val result = detector.detectHighestId() - // then - assertThat(result).hasMessage("Title dump did not contain any anime id.") + // then + assertThat(result).isEqualTo(5) + } } } + @Nested + inner class DownloadOncePerDayTests { + + @Test + fun `keeps the downloaded dump in the download control state directory`() { + tempDirectory { + // given + val detector = detector(tempDir, clientReturning(smallDump)) + + // when + detector.allIds() + + // then + assertThat(tempDir.resolve(DUMP_FILE_NAME).readBytes()).isEqualTo(smallDump) + } + } + + @Test + fun `uses the kept dump without downloading if it is younger than one day`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofHours(23)))) + + val detector = detector(tempDir, notInvokedClient()) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + } + } + + @Test + fun `downloads again if the kept dump is one day old`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(1)))) + + val newDump = "1|1|x-jat|First\n7|1|x-jat|Seventh\n".toByteArray() + val detector = detector(tempDir, clientReturning(newDump)) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 7) + assertThat(file.readBytes()).isEqualTo(newDump) + } + } + + @Test + fun `uses the kept dump if the download fails`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(3)))) + + val detector = detector(tempDir, failingClient()) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + } + } + + @Test + fun `doesn't replace the kept dump with a download without ids`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(3)))) + + val detector = detector(tempDir, clientReturning("maintenance".toByteArray())) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + assertThat(file.readBytes()).isEqualTo(smallDump) + } + } + + @Test + fun `throws if the download fails and there is no kept dump`() { + tempDirectory { + // given + val detector = detector(tempDir, failingClient()) + + // when + val result = exceptionExpected { + detector.allIds() + } + + // then + assertThat(result).hasMessage("unreachable") + } + } + } + + private fun detector( + tempDir: Directory, + httpClient: HttpClient, + deadEntriesAccessor: DeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { } + }, + highestIdAlreadyInDataset: Int = 0, + metaDataProviderConfig: MetaDataProviderConfig = AnidbConfig, + ): AnidbTitlesDumpHighestIdDetector { + val testAppConfig = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = tempDir + override fun clock(): Clock = Clock.fixed(now, UTC) + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = highestIdAlreadyInDataset + } + + return AnidbTitlesDumpHighestIdDetector( + appConfig = testAppConfig, + httpClient = httpClient, + deadEntriesAccessor = deadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + metaDataProviderConfig = metaDataProviderConfig, + ) + } + private fun clientReturning(body: ByteArray): HttpClient = object: HttpClient { override suspend fun get(url: URL, headers: Map>): HttpResponse { return HttpResponse(200, body) @@ -73,4 +317,16 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this } + + private fun notInvokedClient(): HttpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + } + + private fun failingClient(): HttpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = throw IOException("unreachable") + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + } } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt new file mode 100644 index 00000000..8b6e9510 --- /dev/null +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt @@ -0,0 +1,109 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.DefaultAnidbPendingEntries.Companion.FILE_NAME +import io.github.manamiproject.modb.core.extensions.Directory +import io.github.manamiproject.modb.test.tempDirectory +import org.assertj.core.api.Assertions.assertThat +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.io.path.readLines +import kotlin.io.path.writeText +import kotlin.test.Test + +internal class DefaultAnidbPendingEntriesTest { + + private fun testAppConfig(dir: Directory, day: String = "2026-10-29") = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dir + override fun clock(): Clock = Clock.fixed(Instant.parse("${day}T12:00:00Z"), UTC) + } + + @Test + fun `keeps added entries in the download control state directory`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + + // when + pendingEntries.add(20227) + pendingEntries.add(19000) + + // then + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("19000 2026-10-29", "20227 2026-10-29") + assertThat(DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)).all()) + .containsExactlyInAnyOrder(19000, 20227) + } + } + + @Test + fun `removes an entry`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + pendingEntries.add(19000) + + // when + pendingEntries.remove(20227) + + // then + assertThat(pendingEntries.all()).containsExactly(19000) + } + } + + @Test + fun `removes entries which the title dump no longer names`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + pendingEntries.add(19000) + + // when + pendingEntries.removeAllNotIn(setOf(20227, 1)) + + // then + assertThat(pendingEntries.all()).containsExactly(20227) + } + } + + @Test + fun `selects the entries which have been checked longest ago and marks them as checked today`() { + tempDirectory { + // given + tempDir.resolve(FILE_NAME).writeText(""" + # comment + 100 2026-10-20 + 200 2026-10-01 + 300 2026-10-25 + """.trimIndent()) + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + + // when + val result = pendingEntries.selectForRecheck(2) + + // then + assertThat(result).containsExactlyInAnyOrder(100, 200) + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("100 2026-10-29", "200 2026-10-29", "300 2026-10-25") + } + } + + @Test + fun `selects nothing if no recheck is configured`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + + // when + val result = pendingEntries.selectForRecheck(0) + + // then + assertThat(result).isEmpty() + } + } +} diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt new file mode 100644 index 00000000..c96238bb --- /dev/null +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt @@ -0,0 +1,67 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.DefaultAnidbRestrictedEntries.Companion.FILE_NAME +import io.github.manamiproject.modb.core.extensions.Directory +import io.github.manamiproject.modb.test.tempDirectory +import org.assertj.core.api.Assertions.assertThat +import kotlin.io.path.readLines +import kotlin.test.Test + +internal class DefaultAnidbRestrictedEntriesTest { + + private fun testAppConfig(dir: Directory) = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dir + } + + @Test + fun `keeps added entries in the download control state directory`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + + // when + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // then + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("1202", "3531") + assertThat(DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)).all()) + .containsExactlyInAnyOrder(1202, 3531) + } + } + + @Test + fun `removes an entry`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // when + restrictedEntries.remove(3531) + + // then + assertThat(restrictedEntries.all()).containsExactly(1202) + } + } + + @Test + fun `removes entries which the title dump no longer names`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // when + restrictedEntries.removeAllNotIn(setOf(3531, 1)) + + // then + assertThat(restrictedEntries.all()).containsExactly(3531) + } + } +} diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt index 127b9a16..dcd2eb93 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt @@ -21,10 +21,16 @@ import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatNoException import org.junit.jupiter.api.Nested +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC import kotlin.test.Test internal class LivechartCrawlerTest { + // Lies after every newest year of these tests, so only "tba" is rechecked. + private val farFuture = Clock.fixed(Instant.parse("3000-01-01T00:00:00Z"), UTC) + @Nested inner class StartTests { @@ -69,6 +75,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = TestDownloader, @@ -138,6 +145,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -221,6 +229,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -270,9 +279,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { @@ -294,6 +304,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -343,9 +354,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { @@ -370,6 +382,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -420,9 +433,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { @@ -444,6 +458,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -516,6 +531,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -529,6 +545,82 @@ internal class LivechartCrawlerTest { } } + @Test + fun `rechecks the pages from the current season on and tba on every run without memorizing them`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object: Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val memorized = mutableListOf() + val testLastPageMemorizer = object: LastPageMemorizer by TestLastPageMemorizerString { + override suspend fun retrieveLastPage(): String = "winter-2027" + override suspend fun memorizeLastPage(page: String) { + memorized.add(page) + } + } + + val testNewestYearDetector = object: HighestIdDetector by TestHighestIdDetector { + override suspend fun detectHighestId(): Int = 2027 + } + + val requested = mutableListOf() + val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { + override suspend fun idDownloadList(page: String): List { + requested.add(page) + return when (page) { + "fall-2026" -> listOf("new-1", "new-2") + "tba" -> listOf("new-2", "new-3") + else -> emptyList() + } + } + } + + val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val downloadedEntries = mutableListOf() + val testDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + downloadedEntries.add(id) + return id + } + } + + val crawler = LivechartCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloadControlStateScheduler = testDownloadControlStateScheduler, + deadEntriesAccessor = TestDeadEntriesAccessor, + lastPageMemorizer = testLastPageMemorizer, + newestYearDetector = testNewestYearDetector, + clock = { Clock.fixed(Instant.parse("2026-10-30T12:00:00Z"), UTC) }, + paginationIdRangeSelector = testPaginationIdRangeSelector, + alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, + downloader = testDownloader, + ) + + // when + crawler.start() + + // then + assertThat(requested).containsExactly("fall-2026", "winter-2027", "spring-2027", "summer-2027", "fall-2027", "tba") + assertThat(downloadedEntries).containsExactly("new-1", "new-2", "new-3") + assertThat(memorized).isEmpty() + } + } + @Test fun `resumes on last memorized page`() { tempDirectory { @@ -581,6 +673,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -651,6 +744,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = testDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -718,6 +812,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = testDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -776,6 +871,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = TestDownloader, diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt index 65d48df0..104df24d 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt @@ -49,6 +49,36 @@ internal class LivechartNewestYearDetectorTest { } } + @Test + fun `correctly extracts highest id from the layout of october 2026`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartNewestYearDetectorConfig { + override fun buildDataDownloadLink(id: String): URI = URI("http://localhost:8080/highest-id") + } + + val testHttpClient = object: HttpClient by TestHttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse { + return HttpResponse( + code = 200, + body = loadTestResource("crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html"), + ) + } + } + + val livechartHighestIdDetector = LivechartNewestYearDetector( + metaDataProviderConfig = testMetaDataProviderConfig, + httpClient = testHttpClient, + ) + + // when + val result = livechartHighestIdDetector.detectHighestId() + + // then + assertThat(result).isEqualTo(2027) + } + } + @Test fun `throws exception, because the element couldn't be found`() { runTest { @@ -84,7 +114,7 @@ internal class LivechartNewestYearDetectorTest { } // then - assertThat(result).hasMessage("Unable to extract headers.") + assertThat(result).hasMessage("No list of years found on [http://localhost:8080/highest-id], page title [none]. Most likely a block or maintenance page, or the layout has changed.") } } } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt index 542771d8..d932b785 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt @@ -77,6 +77,44 @@ internal class LivechartPaginationIdRangeSelectorTest { } } + @Test + fun `correctly extracts page content of the layout of october 2026`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartPaginationIdRangeSelectorConfig { + override fun isTestContext(): Boolean = true + } + + val testHttpClient = object: HttpClient by TestHttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = HttpResponse( + code = 200, + body = loadTestResource("crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html") + ) + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val livechartPaginationIdRangeSelector = LivechartPaginationIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + httpClient = testHttpClient, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, + ) + + // when + val result = livechartPaginationIdRangeSelector.idDownloadList("fall-2026") + + // then + assertThat(result).containsExactlyInAnyOrder("13226", "13232", "11748") + } + } + @Test fun `removes anime not scheduled for the current week`() { runTest { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt index 1be69cfe..44bd82c2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt @@ -846,6 +846,108 @@ internal class DefaultDeadEntriesAccessorTest { } } + @Nested + inner class ReplaceDeadEntriesTests { + + @Test + fun `replaces the list, writes the file and removes the dcs entries of new dead entries only`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnidbConfig) + } + + val removedFromDcs = mutableListOf() + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun removeDeadEntry(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId) { + removedFromDcs.add(animeId) + } + } + + val testDeserializer = object: Deserializer by TestDeserializer() { + override suspend fun deserialize(source: RegularFile): DeadEntries = DeadEntries( + `$schema` = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/refs/heads/master/dead-entries/dead-entries.schema.json"), + lastUpdate = "2024-08-01", + deadEntries = listOf("111", "222"), + ) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + jsonDeserializer = testDeserializer, + downloadControlStateAccessor = testDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("222", "333", "444"), AnidbConfig) + + // then + assertThat(deadEntriesAccessor.fetchDeadEntries(AnidbConfig)).containsExactlyInAnyOrder("222", "333", "444") + assertThat(removedFromDcs).containsExactlyInAnyOrder("333", "444") + val result = FromRegularFileDeserializer( + deserializer = DeadEntriesFromInputStreamDeserializer.instance).deserialize( + deadEntriesAccessor.deadEntriesFile(AnidbConfig, JSON_MINIFIED), + ).deadEntries + assertThat(result).containsExactlyInAnyOrder("222", "333", "444") + } + } + + @Test + fun `doesn't write the file if nothing changed`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnidbConfig) + } + + val testDeserializer = object: Deserializer by TestDeserializer() { + override suspend fun deserialize(source: RegularFile): DeadEntries = DeadEntries( + `$schema` = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/refs/heads/master/dead-entries/dead-entries.schema.json"), + lastUpdate = "2024-08-01", + deadEntries = listOf("111"), + ) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + jsonDeserializer = testDeserializer, + downloadControlStateAccessor = TestDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("111"), AnidbConfig) + + // then + assertThat(deadEntriesAccessor.deadEntriesFile(AnidbConfig, JSON_MINIFIED)).doesNotExist() + assertThat(deadEntriesAccessor.fetchDeadEntries(AnidbConfig)).containsExactly("111") + } + } + + @Test + fun `does nothing for a metadata provider without dead entry files`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnisearchConfig) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + downloadControlStateAccessor = TestDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("111"), AnisearchConfig) + + // then + assertThat(tempDir.resolve("dead-entries")).doesNotExist() + } + } + } + @Nested inner class DetermineDeadEntriesTests { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt index a7061ca4..5d49682b 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt @@ -1162,6 +1162,49 @@ internal class DefaultDownloadControlStateAccessorTest { } } + @Test + fun `refreshes the data of an entry updated this week already, but keeps its schedule`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun metaDataProviderConfigurations(): Set = setOf(AnilistConfig) + override fun downloadControlStateDirectory(): Directory = tempDir + } + + val firstUpdate = DownloadControlStateEntry( + _weeksWihoutChange = 3, + _lastDownloaded = WeekOfYear.currentWeek(), + _nextDownload = WeekOfYear.currentWeek().plusWeeks(3), + _anime = TestAnimeRawObjects.DefaultAnime.obj, + ) + + val secondUpdate = DownloadControlStateEntry( + _weeksWihoutChange = 4, + _lastDownloaded = WeekOfYear.currentWeek(), + _nextDownload = WeekOfYear.currentWeek().plusWeeks(4), + _anime = TestAnimeRawObjects.AllPropertiesSet.obj, + ) + + tempDir.resolve(AnilistConfig.hostname()).createDirectory() + + val defaultDownloadControlStateAccessor = DefaultDownloadControlStateAccessor( + appConfig = testAppConfig, + mergeLockAccess = TestMergeLockAccessor, + ) + defaultDownloadControlStateAccessor.createOrUpdate(AnilistConfig, "177191", firstUpdate) + + // when + val result = defaultDownloadControlStateAccessor.createOrUpdate(AnilistConfig, "177191", secondUpdate) + + // then + assertThat(result).isTrue() + val stored = defaultDownloadControlStateAccessor.dcsEntry(AnilistConfig, "177191") + assertThat(stored.anime).isEqualTo(TestAnimeRawObjects.AllPropertiesSet.obj) + assertThat(stored.weeksWihoutChange).isEqualTo(3) + assertThat(stored.nextDownload).isEqualTo(WeekOfYear.currentWeek().plusWeeks(3)) + } + } + @Test fun `triggers initialization if necessary`() { tempDirectory { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt index 51db9fe6..33bdf84c 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt @@ -30,6 +30,7 @@ import io.github.manamiproject.modb.test.tempDirectory import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested import java.net.URI +import java.util.concurrent.atomic.AtomicInteger import java.time.Clock import java.time.Instant import java.time.ZoneId @@ -472,6 +473,53 @@ internal class DefaultDownloadControlStateUpdaterTest { } } + @Test + fun `skips and deletes an empty conv file instead of failing the run`() { + tempDirectory { + // given + val rawdata = tempDir.resolve("rawdata").createDirectory() + val dcs = tempDir.resolve("dcs").createDirectory() + + val anime = AnimeRaw( + _title = "Shin Seiki Evangelion Movie: THE END OF EVANGELION", + _sources = hashSetOf(URI("https://anilist.co/anime/32")), + ) + Json.toJson(anime).writeToFile(rawdata.resolve("32.$CONVERTED_FILE_SUFFIX").createFile()) + val emptyFile = rawdata.resolve("64.$CONVERTED_FILE_SUFFIX").createFile() + + val testAppConfig = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dcs + override fun metaDataProviderConfigurations(): Set = setOf(AnilistConfig) + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = rawdata + override fun findMetaDataProviderConfig(host: Hostname): MetaDataProviderConfig = AnilistConfig + override fun clock(): Clock = Clock.fixed(Instant.parse("2021-11-02T17:55:43.035Z"), ZoneId.systemDefault()) + } + + val updatedIds = mutableListOf() + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun allDcsEntries(metaDataProviderConfig: MetaDataProviderConfig): List = emptyList() + override fun downloadControlStateDirectory(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + override suspend fun dcsEntryExists(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId): Boolean = false + override suspend fun createOrUpdate(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId, downloadControlStateEntry: DownloadControlStateEntry): Boolean { + updatedIds.add(animeId) + return true + } + } + + val downloadControlStateUpdater = DefaultDownloadControlStateUpdater( + appConfig = testAppConfig, + downloadControlStateAccessor = testDownloadControlStateAccessor, + ) + + // when + downloadControlStateUpdater.updateAll() + + // then + assertThat(updatedIds).containsExactly("32") + assertThat(emptyFile).doesNotExist() + } + } + @Test fun `creates dcs file if it doesn't exist`() { tempDirectory { @@ -694,7 +742,7 @@ internal class DefaultDownloadControlStateUpdaterTest { override fun clock(): Clock = Clock.fixed(Instant.parse("2021-11-02T17:55:43.035Z"), ZoneId.systemDefault()) } - var invocations = 0 + val invocations = AtomicInteger(0) var hasBeenInvoked = false val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { override suspend fun allDcsEntries(metaDataProviderConfig: MetaDataProviderConfig): List = emptyList() @@ -705,7 +753,7 @@ internal class DefaultDownloadControlStateUpdaterTest { } override suspend fun dcsEntryExists(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId): Boolean = false override suspend fun createOrUpdate(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId, downloadControlStateEntry: DownloadControlStateEntry): Boolean { - invocations++ + invocations.incrementAndGet() return true } } @@ -719,7 +767,7 @@ internal class DefaultDownloadControlStateUpdaterTest { downloadControlStateUpdater.updateAll() // then - assertThat(invocations).isEqualTo(2) + assertThat(invocations.get()).isEqualTo(2) assertThat(hasBeenInvoked).isTrue() } } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt index 920f0c08..e52a8cce 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt @@ -133,6 +133,35 @@ internal class ClearanceHttpClientTest { } } + @Test + fun `closes every refusal it doesn't return`() { + runBlocking { + // given + val responses = mutableListOf() + val codes = mutableListOf(403, 403, 200) + val client = ClearanceHttpClient( + clearance = clearanceWith(solved), + httpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse { + return HttpResponse(codes.removeFirst(), "body").also { responses.add(it) } + } + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + }, + ) + + // when + val result = client.get(URI("https://anidb.net/anime/23").toURL()) + + // then + assertThat(result.code).isEqualTo(200) + assertThat(responses).hasSize(3) + assertThat(responses[0].isBodyInputStreamExhausted()).isTrue() + assertThat(responses[1].isBodyInputStreamExhausted()).isTrue() + assertThat(result.isBodyInputStreamExhausted()).isFalse() + } + } + @Test fun `renews the clearance once when the site refuses it`() { runBlocking { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt index cb780047..1d7369a2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt @@ -11,6 +11,8 @@ import io.github.manamiproject.modb.test.tempDirectory import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.assertThrows +import java.net.ConnectException import java.net.URI import java.net.URL import java.nio.file.Path @@ -48,6 +50,33 @@ internal class CloudflareClearanceTest { } } + @Test + fun `names the challenge solver if it cannot be reached`() { + runBlocking { + // given + val clearance = CloudflareClearance( + configRegistry = registry(file = null), + httpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { + throw ConnectException("Failed to connect to localhost/127.0.0.1:8191") + } + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + }, + hostname = "anidb.net", + ) + + // when + val result = assertThrows { + runBlocking { clearance.refresh(URI("https://anidb.net/anime/23").toURL()) } + } + + // then + assertThat(result).hasMessage("Failed to connect to the challenge solver [localhost/127.0.0.1:8191]") + assertThat(result.cause).isInstanceOf(ConnectException::class.java) + } + } + @Test fun `holds a clearance per route`() { runBlocking { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt index 8e07f426..642b8907 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt @@ -150,7 +150,7 @@ internal class IfconfigKtTest { // given val receivedCommand = mutableListOf() val testCommandExecutor = object : CommandExecutor { - override var config: CommandLineConfig = CommandLineConfig() + override var config: CommandLineConfig = CommandLineConfig(useSudo = true) override fun executeCmd(command: List): String { receivedCommand.addAll(command) return loadTestResource("network/IfconfigKtTest/device_up.txt") @@ -176,7 +176,7 @@ internal class IfconfigKtTest { // given val receivedCommand = mutableListOf() val testCommandExecutor = object : CommandExecutor { - override var config: CommandLineConfig = CommandLineConfig() + override var config: CommandLineConfig = CommandLineConfig(useSudo = true) override fun executeCmd(command: List): String { receivedCommand.addAll(command) return loadTestResource("network/IfconfigKtTest/device_down.txt") @@ -197,6 +197,56 @@ internal class IfconfigKtTest { ) } + @Test + fun `up a specific device without sudo if sudo is disabled`() { + // given + val receivedCommand = mutableListOf() + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig(useSudo = false) + override fun executeCmd(command: List): String { + receivedCommand.addAll(command) + return loadTestResource("network/IfconfigKtTest/device_up.txt") + } + } + + // when + ifconfig("en1", "up") { + commandExecutor = testCommandExecutor + } + + // then + assertThat(receivedCommand).containsExactly( + "ifconfig", + "en1", + "up", + ) + } + + @Test + fun `down a specific device without sudo if sudo is disabled`() { + // given + val receivedCommand = mutableListOf() + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig(useSudo = false) + override fun executeCmd(command: List): String { + receivedCommand.addAll(command) + return loadTestResource("network/IfconfigKtTest/device_down.txt") + } + } + + // when + ifconfig("en1", "down") { + commandExecutor = testCommandExecutor + } + + // then + assertThat(receivedCommand).containsExactly( + "ifconfig", + "en1", + "down", + ) + } + @Test fun `calls ifconfig without parameters if asked for password`() { // given diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt index 63e53958..d01ea94a 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt @@ -106,6 +106,7 @@ internal class LinuxNetworkControllerTest { appConfig = testAppConfig, kotlinCommandExecutor = testCommandExecutor, ) + linuxNetworkController.sudoPasswordValue = "password" // when val result = linuxNetworkController.restartAsync().await() @@ -127,6 +128,70 @@ internal class LinuxNetworkControllerTest { } } + @Test + fun `restarts the network controller without sudo if no sudo password is set`() { + runTest { + // given + var clock = Clock.fixed(Instant.parse("2021-01-31T16:02:42.00Z"), UTC) + + val testAppConfig = object: Config by TestAppConfig { + override fun isTestContext(): Boolean = true + override fun clock(): Clock = clock + } + + val invocations = mutableListOf>() + var numberOfChecks = 0 + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig() + override fun executeCmd(command: List): String { + invocations.add(command) + clock = Clock.fixed(clock.instant().plus(1, ChronoUnit.MINUTES), UTC) + + return when { + command.containsExactlyInTheSameOrder(listOf("ifconfig")) -> loadTestResource("network/LinuxNetworkControllerTest/all_devices.txt") + command.containsExactlyInTheSameOrder(listOf("ifconfig", "en1")) -> { + numberOfChecks++ + + when { + numberOfChecks < 3 -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + numberOfChecks == 3 -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + numberOfChecks in 4..6 -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + numberOfChecks == 7 -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + else -> shouldNotBeInvoked() + } + } + command.any { it.contains("up") } -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + command.any { it.contains("down") } -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + else -> shouldNotBeInvoked() + } + } + } + + val linuxNetworkController = LinuxNetworkController( + appConfig = testAppConfig, + kotlinCommandExecutor = testCommandExecutor, + ) + + // when + val result = linuxNetworkController.restartAsync().await() + + // then + assertThat(result).isTrue() + assertThat(invocations).containsExactly( + listOf("ifconfig"), + listOf("ifconfig", "en1", "down"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1", "up"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + ) + } + } + @Test fun `throws exception if timeout hits while waiting for device changing status`() { runTest { diff --git a/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html b/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html new file mode 100644 index 00000000..0e750c64 --- /dev/null +++ b/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html @@ -0,0 +1,53 @@ +All Seasons | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    All Seasons
    Season Unannounced
    TBA
    2027
    WinterSpring
    SummerFall
    2026 (Current)
    WinterSpring
    SummerFall
    2025
    WinterSpring
    SummerFall
    2024
    WinterSpring
    SummerFall
    2023
    WinterSpring
    SummerFall
    2022
    WinterSpring
    SummerFall
    2021
    WinterSpring
    SummerFall
    2020
    WinterSpring
    SummerFall
    2019
    WinterSpring
    SummerFall
    2018
    WinterSpring
    SummerFall
    2017
    WinterSpring
    SummerFall
    2016
    WinterSpring
    SummerFall
    2015
    WinterSpring
    SummerFall
    2014
    WinterSpring
    SummerFall
    2013
    WinterSpring
    SummerFall
    2012
    WinterSpring
    SummerFall
    2011
    WinterSpring
    SummerFall
    2010
    WinterSpring
    SummerFall
    2009
    WinterSpring
    SummerFall
    2008
    WinterSpring
    SummerFall
    2007
    WinterSpring
    SummerFall
    2006
    WinterSpring
    SummerFall
    2005
    WinterSpring
    SummerFall
    2004
    WinterSpring
    SummerFall
    2003
    WinterSpring
    SummerFall
    2002
    WinterSpring
    SummerFall
    2001
    WinterSpring
    SummerFall
    2000
    WinterSpring
    SummerFall
    1999
    WinterSpring
    SummerFall
    1998
    WinterSpring
    SummerFall
    1997
    WinterSpring
    SummerFall
    1996
    WinterSpring
    SummerFall
    1995
    WinterSpring
    SummerFall
    1994
    WinterSpring
    SummerFall
    1993
    WinterSpring
    SummerFall
    1992
    WinterSpring
    SummerFall
    1991
    WinterSpring
    SummerFall
    1990
    WinterSpring
    SummerFall
    1989
    WinterSpring
    SummerFall
    1988
    WinterSpring
    SummerFall
    1987
    WinterSpring
    SummerFall
    1986
    WinterSpring
    SummerFall
    1985
    WinterSpring
    SummerFall
    1984
    WinterSpring
    SummerFall
    1983
    WinterSpring
    SummerFall
    1982
    WinterSpring
    SummerFall
    1981
    WinterSpring
    SummerFall
    1980
    WinterSpring
    SummerFall
    1979
    WinterSpring
    SummerFall
    1978
    WinterSpring
    SummerFall
    1977
    WinterSpring
    SummerFall
    1976
    WinterSpring
    SummerFall
    1975
    WinterSpring
    SummerFall
    1974
    WinterSpring
    SummerFall
    1973
    WinterSpring
    SummerFall
    1972
    WinterSpring
    SummerFall
    1971
    WinterSpring
    SummerFall
    1970
    WinterSpring
    SummerFall
    1969
    WinterSpring
    SummerFall
    1968
    WinterSpring
    SummerFall
    1967
    WinterSpring
    SummerFall
    1966
    WinterSpring
    SummerFall
    1965
    WinterSpring
    SummerFall
    1964
    WinterSpring
    SummerFall
    1963
    WinterSpring
    SummerFall
    1962
    WinterSpring
    SummerFall
    1961
    WinterSpring
    SummerFall
    1960
    WinterSpring
    SummerFall
    1958
    WinterSpring
    SummerFall
    1957
    WinterSpring
    SummerFall
    1956
    WinterSpring
    SummerFall
    1946
    WinterSpring
    SummerFall
    1944
    WinterSpring
    SummerFall
    1943
    WinterSpring
    SummerFall
    1942
    WinterSpring
    SummerFall
    1936
    WinterSpring
    SummerFall
    1935
    WinterSpring
    SummerFall
    1934
    WinterSpring
    SummerFall
    1932
    WinterSpring
    SummerFall
    1931
    WinterSpring
    SummerFall
    1929
    WinterSpring
    SummerFall
    1917
    WinterSpring
    SummerFall
    1907
    WinterSpring
    SummerFall
    + + + + + + + + + + + \ No newline at end of file diff --git a/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html b/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html new file mode 100644 index 00000000..ed76f088 --- /dev/null +++ b/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html @@ -0,0 +1,12 @@ + + + +Fall 2026 Anime Chart - All | LiveChart.me + + +
    +
    +
    +
    + + diff --git a/livechart/build.gradle.kts b/livechart/build.gradle.kts index 59fde0b7..0a0b0fef 100644 --- a/livechart/build.gradle.kts +++ b/livechart/build.gradle.kts @@ -44,6 +44,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -51,12 +53,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt b/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt index d4a98e23..6759e589 100644 --- a/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt +++ b/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt @@ -54,6 +54,18 @@ public class LivechartAnimeConverter( "sourceDiv" to "//div[@data-anime-details-id]/@data-anime-details-id", "sourceMeta" to "//meta[@property='og:url']/@content", "studios" to "//a[contains(@href, 'studios')]/text()", + // Layout since the redesign of october 2026: the value sits in the element following its label. The keys + // above stay as fallback for pages in the earlier layout. + "typeValue" to "//div[text()='Format']/following-sibling::div[1]/text()", + "statusValue" to "//div[text()='Status']/following-sibling::*[1]/text()", + "durationValue" to "//div[text()='Run time']/following-sibling::div[1]/text()", + "seasonValue" to "//div[text()='Season']/following-sibling::a[1]/text()", + "yearValue" to "//div[text()='Premiere']/following-sibling::*[1]/text()", + "episodesDetails" to "//div[@data-anime-details-target='viewingProgression']/@data-viewing-progression-anime-id", + "episodesTotal" to "//div[@data-anime-details-target='viewingProgression']/@data-viewing-progression-total-count-value", + "episodesNext" to "//div[a[contains(@href,'/schedules/')]]//strong[starts-with(text(),'EP')]/text()", + "relatedAnimeCards" to "//article[@data-related-anime-target='anime']//a/@href", + "tagChips" to "//div[text()='Tags']/following-sibling::div[1]//a/text()", )) val jsonld = data.listNotNull("jsonld").first() @@ -104,8 +116,14 @@ public class LivechartAnimeConverter( var episodes = jsonData.intOrDefault("numberOfEpisodes") var isTableValueUnknown = false - // regular value in data table - if (episodes == 0) { + // number of episodes of the details card, layout since october 2026 + if (episodes == 0 && !data.notFound("episodesDetails")) { + episodes = data.firstText("episodesTotal").toIntOrNull() ?: 0 + isTableValueUnknown = episodes == 0 + } + + // regular value in data table, earlier layout + if (episodes == 0 && data.notFound("episodesDetails")) { val episodesValue = data.stringOrDefault("episodesDiv", "?") .trim() .split('/') @@ -120,7 +138,7 @@ public class LivechartAnimeConverter( // current episode from table if the anime is ongoing if (episodes == 0) { - episodes = data.stringOrDefault("episodesCountdown") + episodes = data.firstText("episodesNext", "episodesCountdown") .remove("EP") .trim() .toIntOrNull() @@ -138,7 +156,7 @@ public class LivechartAnimeConverter( } private fun extractType(data: ExtractionResult): AnimeType { - val value = data.string("type") + val value = data.firstText("typeValue", "type") return when(value.trim().lowercase()) { "movie" -> MOVIE @@ -176,7 +194,7 @@ public class LivechartAnimeConverter( } private fun extractStatus(data: ExtractionResult): AnimeStatus { - val statusString = data.stringOrDefault("status") + val statusString = data.firstText("statusValue", "status") return when (statusString.trim().lowercase()){ "not yet released" -> UPCOMING @@ -187,7 +205,7 @@ public class LivechartAnimeConverter( } private fun extractDuration(data: ExtractionResult): Duration { - val durationString = data.stringOrDefault("duration").trim() + val durationString = data.firstText("durationValue", "duration").trim() val seconds = """(\d+ ?[aA-zZ]+)+""".toRegex() .findAll(durationString) @@ -211,10 +229,11 @@ public class LivechartAnimeConverter( } private fun extractAnimeSeason(data: ExtractionResult): AnimeSeason { - val splitSeasonString = if (data.notFound("season")) { + val seasonText = data.firstText("seasonValue", "season") + val splitSeasonString = if (seasonText.isBlank()) { listOf(EMPTY) } else { - data.listNotNull("season").first().remove("Season ").split(' ') + seasonText.remove("Season ").split(' ') } val seasonString = splitSeasonString.first().trim().lowercase() @@ -229,7 +248,7 @@ public class LivechartAnimeConverter( val year = if (splitSeasonString.size == 2) { YEAR_REGEX.find(splitSeasonString[1])?.value?.trim()?.ifBlank { "0" }?.toIntOrNull() ?: 0 } else { - YEAR_REGEX.find(data.stringOrDefault("year"))?.value?.trim()?.toIntOrNull() ?: 0 + YEAR_REGEX.find(data.firstText("yearValue", "year"))?.value?.trim()?.toIntOrNull() ?: 0 } return AnimeSeason( @@ -261,14 +280,12 @@ public class LivechartAnimeConverter( } private fun extractRelatedAnime(data: ExtractionResult): HashSet { - return if (data.notFound("relatedAnime")) { - hashSetOf() - } else { - data.listNotNull("relatedAnime") - .map { it.remove("/anime/") } - .map { metaDataProviderConfig.buildAnimeLink(it) } - .toHashSet() - } + val key = listOf("relatedAnimeCards", "relatedAnime").firstOrNull { !data.notFound(it) } ?: return hashSetOf() + + return data.listNotNull(key) + .mapNotNull { ANIME_LINK.find(it.trim())?.groupValues?.get(1) } + .map { metaDataProviderConfig.buildAnimeLink(it) } + .toHashSet() } private fun extractTags(jsonData: ExtractionResult, data: ExtractionResult): HashSet { @@ -280,9 +297,10 @@ public class LivechartAnimeConverter( .toHashSet() } - if (!data.notFound("tags")) { - data.listNotNull("tags") + listOf("tagChips", "tags").firstOrNull { !data.notFound(it) }?.let { key -> + data.listNotNull(key) .map { it.trim().lowercase() } + .filter { it.isNotBlank() } .forEach { tags.add(it) } } @@ -315,7 +333,7 @@ public class LivechartAnimeConverter( if (!data.notFound("studios")) { data.listNotNull("studios") .map { it.normalize().lowercase() } - .filterNot { it == "studios" } + .filterNot { it == "studios" || it.isBlank() } .forEach { studios.add(it) } } @@ -326,6 +344,16 @@ public class LivechartAnimeConverter( private const val LARGE_PICTURE_INDICATOR = "large.jpg" private const val SMALL_PICTURE_INDICATOR = "small.jpg" private val YEAR_REGEX = """\d{4}""".toRegex() + private val ANIME_LINK = """^(?:https?://(?:www\.)?livechart\.me)?/anime/(\d+)/?$""".toRegex() + + /** + * Text of the first key which has been found and isn't blank. A key can hold one value or a list of values. + */ + private fun ExtractionResult.firstText(vararg keys: String): String = keys.asSequence() + .filterNot { notFound(it) } + .map { listNotNull(it) { value -> value }.joinToString(" ").trim() } + .firstOrNull { it.isNotBlank() } + ?: EMPTY /** * Singleton of [LivechartAnimeConverter] diff --git a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt index b7527f4e..8585012f 100644 --- a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt +++ b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt @@ -1385,6 +1385,86 @@ internal class LivechartAnimeConverterTest { } } + private fun converter() = LivechartAnimeConverter( + metaDataProviderConfig = object : MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = LivechartConfig.hostname() + override fun buildAnimeLink(id: AnimeId): URI = LivechartConfig.buildAnimeLink(id) + override fun fileSuffix(): FileSuffix = LivechartConfig.fileSuffix() + }, + ) + + @Nested + inner class LayoutOfOctober2026Tests { + + @Test + fun `finished anime`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/12345.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.type).isEqualTo(TV) + assertThat(result.status).isEqualTo(FINISHED) + assertThat(result.episodes).isEqualTo(12) + assertThat(result.duration).isEqualTo(Duration(24, MINUTES)) + assertThat(result.animeSeason.season).isEqualTo(SUMMER) + assertThat(result.animeSeason.year).isEqualTo(2026) + assertThat(result.tags).contains("action", "adventure", "fantasy") + assertThat(result.studios).contains("maho film") + assertThat(result.studios).doesNotContain("") + assertThat(result.relatedAnime).isEmpty() + } + } + + @Test + fun `airing sequel`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/11343.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.type).isEqualTo(TV) + assertThat(result.status).isEqualTo(ONGOING) + assertThat(result.episodes).isEqualTo(13) + assertThat(result.animeSeason.season).isEqualTo(FALL) + assertThat(result.animeSeason.year).isEqualTo(2026) + assertThat(result.relatedAnime).containsExactly( + URI("https://livechart.me/anime/10355"), + ) + } + } + + @Test + fun `airing anime without a known number of episodes takes the episode before the next one`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/11428.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.status).isEqualTo(ONGOING) + assertThat(result.episodes).isEqualTo(1751) + assertThat(result.animeSeason.season).isEqualTo(FALL) + assertThat(result.animeSeason.year).isEqualTo(1988) + assertThat(result.relatedAnime).containsExactlyInAnyOrder( + URI("https://livechart.me/anime/11443"), + URI("https://livechart.me/anime/11430"), + URI("https://livechart.me/anime/11397"), + URI("https://livechart.me/anime/7249"), + URI("https://livechart.me/anime/11405"), + ) + } + } + } + @Nested inner class CompanionObjectTests { diff --git a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt index 346e6acf..75ea064f 100644 --- a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt +++ b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt @@ -42,6 +42,10 @@ private val files = mapOf( "LivechartAnimeConverterTest/episodes/number_of_episodes_unknown_but_currently_running.html" to "319", "LivechartAnimeConverterTest/episodes/unknown.html" to "11873", + "LivechartAnimeConverterTest/layout_2026_10/11343.html" to "11343", + "LivechartAnimeConverterTest/layout_2026_10/11428.html" to "11428", + "LivechartAnimeConverterTest/layout_2026_10/12345.html" to "12345", + "LivechartAnimeConverterTest/picture_and_thumbnail/neither_picture_nor_thumbnail.html" to "2301", "LivechartAnimeConverterTest/picture_and_thumbnail/picture_and_thumbnail_available.html" to "3437", diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html new file mode 100644 index 00000000..84dbfa0d --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html @@ -0,0 +1,86 @@ + +The Detective is Already Dead Season 2 | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Tantei wa Mou, Shindeiru. Season2
    English title
    The Detective is Already Dead Season 2
    Native title
    探偵はもう、死んでいる。Season2
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Fall 2026)
    7.9/10
    181 Ratings
    –/10
    You
    EP5 · TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Format
    TV
    Episodes
    ?/13
    Run time
    24m

    After thwarting a hijacking with beautiful, silver-haired detective Siesta, Kimihiko leads a life of globe-trotting and crime-solving—until Siesta dies. A year later, a new case will open old wounds as it threads murder, conspiracy, and his dead partner!

    [Source: Crunchyroll]

    Studio
    ENGI
    Tags
    ComedyDramaMysteryRomanceSequel
    Videos
    Play video: PV 3
    :51
    JAEN
    PV 3
    Play video: PV 2
    :24
    JAEN
    PV 2
    Streams
    Crunchyroll
    Sub & Dub (English, French, German, Hindi, Portuguese (Brazil), Spanish (Latin America))
    Franchise: The Detective is Already Dead
    The Detective is Already Dead
    • TV
    • 2021
    • 7.32
    ?/12
    Show more
    WebsiteOfficial X
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearchKitsu
    + + + + + + + + + + + + \ No newline at end of file diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html new file mode 100644 index 00000000..278ff639 --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html @@ -0,0 +1,86 @@ + +Sore Ike! Anpanman | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Sore Ike! Anpanman
    Native title
    それいけ！アンパンマン
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Fall 1988)
    EP1752 · TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Format
    TV
    Episodes
    ?/–
    Run time
    24m

    One night, a Star of Life falls down the chimney of a bakery nestled deep in the forest, causing the dough in the oven to come to life. The dough becomes Anpanman, a superhero made of anpan (a sweet roll with bean jam filling). Together with his friends, Anpanman fights his rival Baikinman and helps the malnourished.

    [Source: Anime News Network]

    Studio
    TMS Entertainment
    Tags
    ComedyFantasyKids
    Franchise: Anpanman
    Go! Anpanman: The Shining Star's Tear
    • Movie
    • 1989
    ?/1
    Sore Ike! Anpanman: Baikinman no Gyakushuu
    • Movie
    • 1990
    ?/1
    Sore Ike! Anpanman: Omusubiman
    • Movie
    • 1990
    ?/1
    Sore Ike! Anpanman: Minami no Umi wo Sukue!
    • Special
    • 1990
    ?/1
    Sore Ike! Anpanman: Dokin-chan no Dokidoki Calendar
    • Movie
    • 1991
    ?/1
    Show more
    Website
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearchKitsu
    + + + + + + + + + + + + \ No newline at end of file diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html new file mode 100644 index 00000000..5492d4c2 --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html @@ -0,0 +1,86 @@ + +The Insipid Prince's Furtive Grab for The Throne | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Saikyou Degarashi Ouji no Anyaku Teii Arasoi: Munou wo Enjiru SS Rank Ouji wa Koui Keishousen wo Kage kara Shihai suru
    English title
    The Insipid Prince's Furtive Grab for The Throne
    Native title
    最強出涸らし皇子の暗躍帝位争い 無能を演じるSSランク皇子は皇位継承戦を影から支配する
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Summer 2026)
    7.5/10
    832 Ratings
    –/10
    You
    TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Released
    Format
    TV
    Episodes
    ?/12
    Run time
    24m

    To the empire, Prince Arnold is one big joke, a completely inept prince outshone by his more talented twin, Leonard. But in secret, Arnold operates as Silver, one of only five magic-wielding SS-rank adventurers on the continent. As the bloody battle for the throne intensifies after the crown prince’s death, Arnold pulls the strings from the shadows to crown his twin emperor to his own advantage.

    [Source: Crunchyroll]

    Studio
    MAHO FILM
    Tags
    ActionAdventureFantasy
    Videos
    Play video: PV 2
    :51
    JANone
    PV 2
    Play video: PV 1
    :10
    JANone
    PV 1
    Streams
    Crunchyroll
    Sub & Dub (Thai)
    WebsiteOfficial X
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearch
    + + + + + + + + + + + + \ No newline at end of file diff --git a/myanimelist/build.gradle.kts b/myanimelist/build.gradle.kts index 85685815..ec4d2eec 100644 --- a/myanimelist/build.gradle.kts +++ b/myanimelist/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/serde/build.gradle.kts b/serde/build.gradle.kts index 31c1adce..719cb11a 100644 --- a/serde/build.gradle.kts +++ b/serde/build.gradle.kts @@ -44,6 +44,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -51,12 +53,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/simkl/build.gradle.kts b/simkl/build.gradle.kts index f41adaa4..61557106 100644 --- a/simkl/build.gradle.kts +++ b/simkl/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -49,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/test/build.gradle.kts b/test/build.gradle.kts index c91e8f93..03031117 100644 --- a/test/build.gradle.kts +++ b/test/build.gradle.kts @@ -46,6 +46,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -53,12 +55,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) }