From a601cd2b5a199b6f710c720d6c883b03ef5da34b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 23 Oct 2026 04:31:07 +0200 Subject: [PATCH 01/50] anidb: wait at least 2 seconds between API requests --- .../manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index fe828a23..0ca426a2 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -208,7 +208,7 @@ class AnidbCrawler( private const val DEFAULT_MAX_CONNECTION_CHANGES = 5 private const val DEFAULT_ROUTE_FAILURE_WAIT_IN_SECONDS = 5 - private const val DEFAULT_MIN_WAIT_IN_MILLISECONDS = 1500 + private const val DEFAULT_MIN_WAIT_IN_MILLISECONDS = 2000 private const val DEFAULT_MAX_WAIT_IN_MILLISECONDS = 2500 private const val DEFAULT_PAGES_PER_CONNECTION = 0 From d6558341b4797eb187c46540d27ad031bfab6aaa Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 21:04:26 +0100 Subject: [PATCH 02/50] Respect Retry-After, retry AniDB bans with backoff and stop the crawler if the ban persists, AniList minimum wait 0.67 s --- .../modb/anidb/AnidbApiDownloader.kt | 45 ++++++- .../modb/anidb/AnidbBannedException.kt | 9 ++ .../modb/anidb/AnidbApiDownloaderTest.kt | 114 ++++++++++++++++++ .../modb/core/httpclient/DefaultHttpClient.kt | 21 +++- .../modb/core/httpclient/RetryAfter.kt | 40 ++++++ .../modb/core/httpclient/RetryAfterKtTest.kt | 77 ++++++++++++ .../modb/app/crawlers/anidb/AnidbCrawler.kt | 5 +- .../app/crawlers/anilist/AnilistCrawler.kt | 4 +- 8 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt create mode 100644 anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt create mode 100644 core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt create mode 100644 core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt index faa972b3..7b0c62b0 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt @@ -10,7 +10,12 @@ import io.github.manamiproject.modb.core.downloader.Downloader import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.logging.LoggerDelegate +import io.github.manamiproject.modb.core.excludeFromTestContext +import io.github.manamiproject.modb.core.random +import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import kotlin.time.DurationUnit.MILLISECONDS +import kotlin.time.toDuration import java.io.ByteArrayInputStream import java.net.URI import java.util.zip.GZIPInputStream @@ -45,6 +50,34 @@ public class AnidbApiDownloader( override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = withContext(LIMITED_NETWORK) { log.debug { "Downloading [anidbId=$id]" } + var attempt = 0 + + while (true) { + try { + return@withContext downloadOnce(id, onDeadEntry) + } catch (e: AnidbBannedException) { + attempt++ + + if (attempt > MAX_BAN_RETRIES) { + log.error { "Still banned by the API after [$MAX_BAN_RETRIES] retries. Giving up on [anidbId=$id]." } + throw e + } + + // Same waiting time as the retries of the http client. The ban is answered with + // http 200, so the http client itself doesn't see it as a reason to retry. + val waitTime = (random(MIN_BAN_WAIT_IN_MILLISECONDS, MAX_BAN_WAIT_IN_MILLISECONDS) * attempt).toDuration(MILLISECONDS) + log.warn { "Banned by the API. Retry [$attempt/$MAX_BAN_RETRIES] for [anidbId=$id] in [$waitTime]." } + + excludeFromTestContext(metaDataProviderConfig) { + delay(waitTime) + } + } + } + + EMPTY_RESPONSE + } + + private suspend fun downloadOnce(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { val url = URI("${metaDataProviderConfig.buildDataDownloadLink(id)}&client=$client&clientver=$clientver&protover=$PROTOCOL_VERSION").toURL() val response = httpClient.get(url) val body = decompressIfNecessary(response.bodyAsByteArray()) @@ -64,13 +97,17 @@ public class AnidbApiDownloader( if (code in DEAD_ENTRY_CODES || message.lowercase() in DEAD_ENTRY_MESSAGES) { log.info { "Adding [anidbId=$id] to dead entries list, because the API responded with [$message]." } onDeadEntry.invoke(id) - return@withContext EMPTY_RESPONSE + return EMPTY_RESPONSE + } + + if (message.lowercase() == BANNED_MESSAGE) { + throw AnidbBannedException("API responded with error [code=$code], [message=$message] for [anidbId=$id].") } throw IllegalStateException("API responded with error [code=$code], [message=$message] for [anidbId=$id].") } - return@withContext body + return body } private fun decompressIfNecessary(body: ByteArray): String { @@ -88,6 +125,10 @@ public class AnidbApiDownloader( private val log by LoggerDelegate() private const val PROTOCOL_VERSION = 1 + private const val BANNED_MESSAGE = "banned" + private const val MAX_BAN_RETRIES = 5 + private const val MIN_BAN_WAIT_IN_MILLISECONDS = 60000 + private const val MAX_BAN_WAIT_IN_MILLISECONDS = 90000 private const val EMPTY_RESPONSE = "" private const val GZIP_MAGIC_FIRST: Byte = 0x1f private const val GZIP_MAGIC_SECOND: Byte = 0x8b.toByte() diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt new file mode 100644 index 00000000..237eaa55 --- /dev/null +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbBannedException.kt @@ -0,0 +1,9 @@ +package io.github.manamiproject.modb.anidb + +/** + * Thrown if the AniDB API answers with `banned`. + * The API sends the ban with http status 200 and without any information on how long it lasts. + * @since 20.0.0 + * @param message Description including the anime ID which was requested. + */ +public class AnidbBannedException(message: String) : RuntimeException(message) diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt new file mode 100644 index 00000000..212245a1 --- /dev/null +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt @@ -0,0 +1,114 @@ +package io.github.manamiproject.modb.anidb + +import io.github.manamiproject.modb.core.config.ConfigRegistry +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.httpclient.HttpClient +import io.github.manamiproject.modb.core.httpclient.HttpResponse +import io.github.manamiproject.modb.core.httpclient.RequestBody +import io.github.manamiproject.modb.core.httpclient.RetryCase +import io.github.manamiproject.modb.test.exceptionExpected +import io.github.manamiproject.modb.test.shouldNotBeInvoked +import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Nested +import java.net.URL +import kotlin.test.Test + +internal class AnidbApiDownloaderTest { + + @Nested + inner class BannedTests { + + @Test + fun `retries if the API responds with banned and returns the body once the ban is over`() { + runTest { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + when (invocations) { + 1, 2 -> """banned""" + else -> """""" + } + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = downloader.download("1") { shouldNotBeInvoked() } + + // then + assertThat(result).isEqualTo("""""") + assertThat(invocations).isEqualTo(3) + } + } + + @Test + fun `throws AnidbBannedException if the ban is still active after all retries`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """banned""" + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(result).hasMessage("API responded with error [code=500], [message=banned] for [anidbId=1].") + assertThat(invocations).isEqualTo(6) + } + + @Test + fun `other errors are not retried`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """client version missing or invalid""" + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + val result = exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(result).hasMessage("API responded with error [code=302], [message=client version missing or invalid] for [anidbId=1].") + assertThat(invocations).isEqualTo(1) + } + } + + private val testAnidbConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + private val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun string(key: String): String = "test" + } + + private fun respondingWith(body: () -> String) = object: HttpClient { + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun get(url: URL, headers: Map>): HttpResponse = HttpResponse(200, body()) + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = shouldNotBeInvoked() + } +} diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index cb559bf2..fec4a7ae 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -12,6 +12,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.withContext +import kotlin.time.Duration import okhttp3.* import okhttp3.Protocol.HTTP_1_1 as OKHTTP_HTTP_1_1 import okhttp3.Protocol.HTTP_2 as OKHTTP_HTTP_2 @@ -158,6 +159,7 @@ public class DefaultHttpClient( var attempt = 0 var retryCase: RetryCase = NoRetry var responseOrException: Any = HttpResponse(100, EMPTY) + var retryAfter: Duration? = null do { if (retryCase !is NoRetry) { @@ -185,7 +187,7 @@ public class DefaultHttpClient( okhttpClient = currentClient.newBuilder().build() } - wait(request, retryCase, attempt) + wait(request, retryCase, attempt, retryAfter) } responseOrException = safelyExecute { @@ -201,7 +203,10 @@ public class DefaultHttpClient( true -> { log.debug { "[${request.method} ${request.url}] Requires retry." } + retryAfter = null + if (responseOrException is HttpResponse) { + retryAfter = parseRetryAfter(responseOrException.headers) responseOrException.close() } @@ -229,10 +234,18 @@ public class DefaultHttpClient( } @KoverIgnore - private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int) { + private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int, retryAfter: Duration?) { if (!isTestContext) { - log.trace { "Initiating waiting time for retry of [${request.method} ${request.url}]." } - delay(retryCase.waitDuration(attempt)) + val duration = when (retryAfter) { + null -> retryCase.waitDuration(attempt) + else -> { + log.info { "Server asked to retry [${request.method} ${request.url}] after [$retryAfter]." } + retryAfter + } + } + + log.trace { "Initiating waiting time of [$duration] for retry of [${request.method} ${request.url}]." } + delay(duration) } } diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt new file mode 100644 index 00000000..36eb5610 --- /dev/null +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfter.kt @@ -0,0 +1,40 @@ +package io.github.manamiproject.modb.core.httpclient + +import java.time.Clock +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME +import java.time.format.DateTimeParseException +import kotlin.time.Duration +import kotlin.time.Duration.Companion.ZERO +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +/** + * Reads the `Retry-After` header of a response (RFC 9110, section 10.2.3). + * The value is either a number of seconds or an HTTP date. + * @since 20.0.0 + * @param headers Headers of the response. The header name is matched case-insensitive. + * @param clock Used to calculate the time until an HTTP date. + * @return How long to wait before the next request or `null` if the header is missing or can't be read. + * A date in the past results in [ZERO]. + */ +internal fun parseRetryAfter(headers: Map>, clock: Clock = Clock.systemUTC()): Duration? { + val value = headers.entries + .firstOrNull { it.key.equals("retry-after", ignoreCase = true) } + ?.value + ?.firstOrNull() + ?.trim() + ?: return null + + value.toLongOrNull()?.let { secondsToWait -> + return if (secondsToWait < 0) null else secondsToWait.seconds + } + + return try { + val date = ZonedDateTime.parse(value, RFC_1123_DATE_TIME) + val millisToWait = date.toInstant().toEpochMilli() - clock.millis() + if (millisToWait <= 0) ZERO else millisToWait.milliseconds + } catch (e: DateTimeParseException) { + null + } +} diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt new file mode 100644 index 00000000..f286061c --- /dev/null +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/RetryAfterKtTest.kt @@ -0,0 +1,77 @@ +package io.github.manamiproject.modb.core.httpclient + +import org.assertj.core.api.Assertions.assertThat +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.test.Test +import kotlin.time.Duration.Companion.ZERO +import kotlin.time.Duration.Companion.seconds + +internal class RetryAfterKtTest { + + private val clock = Clock.fixed(Instant.parse("2026-10-27T20:00:00Z"), UTC) + + @Test + fun `returns null if the header is missing`() { + // when + val result = parseRetryAfter(mapOf("content-type" to listOf("application/json")), clock) + + // then + assertThat(result).isNull() + } + + @Test + fun `reads delay in seconds`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("42")), clock) + + // then + assertThat(result).isEqualTo(42.seconds) + } + + @Test + fun `header name is matched case-insensitive`() { + // when + val result = parseRetryAfter(mapOf("Retry-After" to listOf(" 7 ")), clock) + + // then + assertThat(result).isEqualTo(7.seconds) + } + + @Test + fun `reads http date`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("Tue, 27 Oct 2026 20:01:30 GMT")), clock) + + // then + assertThat(result).isEqualTo(90.seconds) + } + + @Test + fun `http date in the past results in zero`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("Tue, 27 Oct 2026 19:59:00 GMT")), clock) + + // then + assertThat(result).isEqualTo(ZERO) + } + + @Test + fun `returns null for a value which can't be read`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("soon")), clock) + + // then + assertThat(result).isNull() + } + + @Test + fun `returns null for a negative number`() { + // when + val result = parseRetryAfter(mapOf("retry-after" to listOf("-5")), clock) + + // then + assertThat(result).isNull() + } +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index 0ca426a2..4c48f874 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -4,6 +4,7 @@ import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anidb.AnidbApiDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbBannedException import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config @@ -69,7 +70,9 @@ class AnidbCrawler( hostname = metaDataProviderConfig.hostname(), // Detection is answered below by leaving through another connection. Skipping it here // would burn the run's remaining entries against an address anidb has already refused. - isHandledByCaller = { it is CrawlerDetectedException }, + // A ban which outlasts the downloader's own retries stops the crawler right away for the + // same reason: every further request during a ban can extend it. + isHandledByCaller = { it is CrawlerDetectedException || it is AnidbBannedException }, ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt index 93eb26f6..c004d759 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt @@ -88,14 +88,14 @@ class AnilistCrawler( @KoverIgnore private suspend fun waitAfterDeadEntry() { excludeFromTestContext(metaDataProviderConfig) { - delay(random(1000, 1500).toDuration(MILLISECONDS)) + delay(random(670, 1500).toDuration(MILLISECONDS)) } } @KoverIgnore private suspend fun wait() { excludeFromTestContext(metaDataProviderConfig) { - delay(random(1000, 1500).toDuration(MILLISECONDS)) + delay(random(670, 1500).toDuration(MILLISECONDS)) } } From 4efa9e7a5e21099a47472373b837c21955869ed8 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 21:08:54 +0100 Subject: [PATCH 03/50] Log highest ID and number of IDs left to download at crawler start --- .../modb/app/crawlers/IntegerBasedIdRangeSelector.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt index 1f08fa7d..cbf28aeb 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt @@ -73,17 +73,29 @@ class IntegerBasedIdRangeSelector( log.info { "Rechecking [${recheck.size}] of [${deadEntries.size}] dead entries for [${metaDataProviderConfig.hostname()}]." } } + val beforeDeadEntries = possibleIds.size possibleIds.removeAll(deadEntries - recheck) + val excludedDeadEntries = beforeDeadEntries - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding dead entries." } log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] before excluding entries which are not scheduled for the current week." } + val beforeNotScheduled = possibleIds.size possibleIds.removeAll(downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig).map { it.toInt() }.toSet()) + val excludedNotScheduled = beforeNotScheduled - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding entries which are not scheduled for the current week." } log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] before excluding already downloaded entries." } + val beforeAlreadyDownloaded = possibleIds.size possibleIds.removeAll(alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).map { it.toInt() }.toSet()) + val excludedAlreadyDownloaded = beforeAlreadyDownloaded - possibleIds.size log.debug { "Having [${possibleIds.size}] for [${metaDataProviderConfig.hostname()}] after excluding already downloaded entries." } + log.info { + "[${metaDataProviderConfig.hostname()}] highest ID is [$highestId]. " + + "Skipping [$excludedDeadEntries] dead entries, [$excludedNotScheduled] entries not scheduled for this week " + + "and [$excludedAlreadyDownloaded] already downloaded. [${possibleIds.size}] of [$highestId] IDs left to download." + } + return possibleIds.toList().createShuffledList() } From a54af662e7d46032098cfe798ff827c746aebfae Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 21:57:58 +0100 Subject: [PATCH 04/50] Shorter logger names in log output --- analyzer/src/main/resources/logback.xml | 2 +- app/src/main/resources/logback.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/analyzer/src/main/resources/logback.xml b/analyzer/src/main/resources/logback.xml index e56c26eb..269bba9e 100644 --- a/analyzer/src/main/resources/logback.xml +++ b/analyzer/src/main/resources/logback.xml @@ -1,5 +1,5 @@ - + diff --git a/app/src/main/resources/logback.xml b/app/src/main/resources/logback.xml index 04931f0b..1646138d 100644 --- a/app/src/main/resources/logback.xml +++ b/app/src/main/resources/logback.xml @@ -1,6 +1,6 @@ - + From 458109baa89c52fed5a5b5ea702822f7d1106d01 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 22:19:51 +0100 Subject: [PATCH 05/50] Page-based crawlers: log already downloaded entries and the page they continue from --- .../crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt | 7 +++++++ .../modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt | 7 +++++++ .../modb/app/crawlers/anisearch/AnisearchCrawler.kt | 7 +++++++ .../modb/app/crawlers/livechart/LivechartCrawler.kt | 7 +++++++ .../manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt | 6 ++++++ 5 files changed, 34 insertions(+) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt index bc145472..c80ba68e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animenewsnetwork/AnimenewsnetworkCrawler.kt @@ -93,6 +93,13 @@ class AnimenewsnetworkCrawler( pages = pages.dropWhile { it != lastPage }.drop(1) } + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [${pages.firstOrNull() ?: "-"}] with [${pages.size}] pages left." + } + wait() pages.forEach { page -> diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt index 620d5703..e7ffbc30 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/animeplanet/AnimePlanetCrawler.kt @@ -84,6 +84,13 @@ class AnimePlanetCrawler( val lastPage = lastPageDetector.detectHighestId() val entriesNotScheduledForCurrentWeek = downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig) + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [$firstPage] of [$lastPage]." + } + wait() for (currentPage in firstPage..lastPage) { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt index 8b1a3150..084ce8db 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt @@ -108,6 +108,13 @@ class AnisearchCrawler( val lastPage = lastPageDetector.detectHighestId() val entriesNotScheduledForCurrentWeek = downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig) + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [$firstPage] of [$lastPage]." + } + wait() for (currentPage in firstPage..lastPage) { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt index 6ef49dd6..65b38b2d 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt @@ -92,6 +92,13 @@ class LivechartCrawler( pages = pages.dropWhile { it != lastPage }.drop(1) } + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + + "Continuing from page [${pages.firstOrNull() ?: "-"}] with [${pages.size}] pages left." + } + wait() pages.forEach { page -> diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt index eada26cd..98cd9c75 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/simkl/SimklCrawler.kt @@ -87,6 +87,12 @@ class SimklCrawler( } val maxYear = LocalDate.now(appConfig.clock()).year + 2 + val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size + log.info { + "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. Continuing from year [$startYear] up to [$maxYear]." + } + for (currentYear in startYear..maxYear) { for (currentOffset in 0..Int.MAX_VALUE step 20) { log.debug { "Checking year [$currentYear] with offset [$currentOffset] for [${metaDataProviderConfig.hostname()}]." } From 4da7a7a106a89b14f67e5c0a5ac9abc19eea0c6c Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 22:58:40 +0100 Subject: [PATCH 06/50] Name the site in log messages which didn't --- .../modb/core/httpclient/DefaultHttpClient.kt | 2 +- .../livechart/LivechartPaginationIdRangeSelector.kt | 2 +- .../DefaultDownloadControlStateScheduler.kt | 4 ++-- .../modb/app/network/SuspendableHttpClient.kt | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index fec4a7ae..3b5fcde6 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -167,7 +167,7 @@ public class DefaultHttpClient( if ((responseOrException is HttpResponse) && (responseOrException.code == 103)) { responseOrException.close() - log.warn { "Received HTTP status code 103. Deactivating HTTP/2." } + log.warn { "Received HTTP status code 103 from [${request.url.host}]. Deactivating HTTP/2." } protocols.remove(HTTP_2) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt index dd04ff7c..5f6516af 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt @@ -57,7 +57,7 @@ class LivechartPaginationIdRangeSelector( headers = mapOf("host" to listOf("www.${metaDataProviderConfig.hostname()}")), ).use { response -> if (response.code == 404) { - log.warn { "404 on LivechartPaginationIdRangeSelector" } + log.warn { "[${metaDataProviderConfig.hostname()}] page [$page] doesn't exist (404). Treating it as empty." } return emptyList() } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt index 1cc092c9..c828a7a9 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt @@ -22,7 +22,7 @@ class DefaultDownloadControlStateScheduler( ): DownloadControlStateScheduler { override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are not scheduled to download this week." } + log.info { "Finding all anime which are not scheduled to download this week for [${metaDataProviderConfig.hostname()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isAfterCurrentWeek(it.nextDownload) } @@ -31,7 +31,7 @@ class DefaultDownloadControlStateScheduler( } override suspend fun findEntriesScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are scheduled to download this week." } + log.info { "Finding all anime which are scheduled to download this week for [${metaDataProviderConfig.hostname()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isCurrentWeekOrOlder(it.nextDownload) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt index 3c73609c..9b717e6b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/SuspendableHttpClient.kt @@ -32,12 +32,12 @@ class SuspendableHttpClient( ): HttpClient { override suspend fun get(url: URL, headers: Map>): HttpResponse { - suspendIfNecessary() + suspendIfNecessary(url) return httpClient.get(url, headers) } override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { - suspendIfNecessary() + suspendIfNecessary(url) return httpClient.post(url, requestBody, headers) } @@ -46,9 +46,9 @@ class SuspendableHttpClient( return this } - private suspend fun suspendIfNecessary() = withContext(Default) { + private suspend fun suspendIfNecessary(url: URL) = withContext(Default) { while(!networkController.isNetworkActive() && isActive) { - log.info { "Waiting for network to be active again." } + log.info { "Waiting for network to be active again before requesting [${url.host}]." } wait() } } From f7f38cf8fdca1ab4622090f3cf42dc397ac4f02a Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 23:21:36 +0100 Subject: [PATCH 07/50] Ask for the sudo password only if needed and verify it, log the network controller choice once per site --- .../io/github/manamiproject/modb/app/App.kt | 56 ++++++++++++++++++- .../app/network/LinuxNetworkController.kt | 3 +- .../modb/app/network/NetworkControllers.kt | 28 +++++++++- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 8a260d99..5ccc2faa 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app +import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig import io.github.manamiproject.modb.app.convfiles.DefaultRawFileConversionService @@ -21,6 +22,7 @@ import io.github.manamiproject.modb.app.downloadcontrolstate.DefaultDownloadCont import io.github.manamiproject.modb.app.extensions.alertDeletedAnimeByTitle import io.github.manamiproject.modb.app.fluentapi.* import io.github.manamiproject.modb.app.network.LinuxNetworkController +import io.github.manamiproject.modb.app.network.NetworkControllers import io.github.manamiproject.modb.app.postprocessors.* import io.github.manamiproject.modb.core.coroutines.CoroutineManager.runCoroutine import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK @@ -37,13 +39,20 @@ import kotlin.system.exitProcess private val log = org.slf4j.LoggerFactory.getLogger("App") fun main() = runCoroutine { - LinuxNetworkController.instance.sudoPasswordValue = passwordPrompt() + val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() + + // Only the crawlers of these sites change the outgoing address by restarting the network device. + val sitesChangingAddress = setOf(AnidbConfig.hostname(), AnisearchConfig.hostname()) - disabled + + when { + isRoot() -> log.info("Running as root. No sudo password needed.") + !NetworkControllers.usesNetworkDevice(sitesChangingAddress) -> log.info("Network device won't be restarted. No sudo password needed.") + else -> LinuxNetworkController.instance.sudoPasswordValue = verifiedSudoPassword() + } val rawFileConversionService = DefaultRawFileConversionService.instance rawFileConversionService.start() - val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() - val crawlers = mutableListOf() withContext(LIMITED_NETWORK) { @@ -91,6 +100,47 @@ fun main() = runCoroutine { ).forEach { it.process() } } +@KoverIgnore +private fun isRoot(): Boolean = try { + val process = ProcessBuilder("id", "-u").redirectErrorStream(true).start() + val userId = process.inputStream.bufferedReader().readText().trim() + process.waitFor() == 0 && userId == "0" +} catch (_: Exception) { + false +} + +@KoverIgnore +private fun verifiedSudoPassword(): String { + repeat(MAX_SUDO_ATTEMPTS) { attempt -> + val password = passwordPrompt() + + if (isValidSudoPassword(password)) { + return password + } + + log.warn("Incorrect sudo password [${attempt + 1}/$MAX_SUDO_ATTEMPTS].") + } + + log.error("No valid sudo password after [$MAX_SUDO_ATTEMPTS] attempts. Exiting.") + exitProcess(1) +} + +/** + * `-k` ignores cached credentials so that the password is really checked, `-S` reads it from stdin + * and `-v` only validates without running a command. + */ +@KoverIgnore +private fun isValidSudoPassword(password: String): Boolean = try { + val process = ProcessBuilder("sudo", "-k", "-S", "-p", "", "-v").redirectErrorStream(true).start() + process.outputStream.bufferedWriter().use { it.write(password); it.newLine() } + process.inputStream.bufferedReader().readText() + process.waitFor() == 0 +} catch (_: Exception) { + false +} + +private const val MAX_SUDO_ATTEMPTS = 3 + @KoverIgnore private fun passwordPrompt(): String { val console = System.console() diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt index 562b0a7f..d26c4eac 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkController.kt @@ -124,7 +124,8 @@ class LinuxNetworkController( ifconfig(device, option) { commandExecutor = kotlinCommandExecutor.apply { - config.useSudo = true + // No password means the app runs as root and doesn't need sudo. + config.useSudo = sudoPasswordValue.isNotBlank() config.sudoPassword = sudoPasswordValue } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt index be04f271..a0c2399d 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/NetworkControllers.kt @@ -21,6 +21,7 @@ object NetworkControllers { private val rotatingControllers = mutableMapOf() private val routedControllers = mutableMapOf, RotatingProxyNetworkController>() + private val loggedDecisions = mutableSetOf>() /** * Returns the one rotating controller belonging to a configuration. @@ -100,16 +101,39 @@ object NetworkControllers { return when { rotating.hasProxies() -> rotating canRestartNetworkDevice() -> { - log.info { "Using the network device to change the outgoing address." } + logOncePerSite(hostname, "Using the network device to change the outgoing address.") LinuxNetworkController.instance } else -> { - log.info { "No proxy pool configured and no network device to restart, so the outgoing address cannot be changed on demand." } + logOncePerSite(hostname, "No proxy pool configured and no network device to restart, so the outgoing address cannot be changed on demand.") FixedNetworkController } } } + // Several classes of the same crawler ask for a controller, which would repeat the same message for each of them. + private fun logOncePerSite(hostname: String?, message: String) { + val site = hostname ?: "all sites" + val isNew = synchronized(loggedDecisions) { loggedDecisions.add(site to message) } + + if (isNew) { + log.info { "[$site] $message" } + } + } + + /** + * Tells in advance whether [forDeployment] will hand out [LinuxNetworkController] for any of the given sites, + * which is the only case in which a sudo password is needed. + * @since 20.0.0 + * @param hostnames Sites whose crawlers change the outgoing address on demand. + * @param configRegistry Source of the proxy pool definition. + * @return `true` if at least one of the sites has no proxies and the network device can be restarted. + */ + fun usesNetworkDevice(hostnames: Collection, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance): Boolean { + val withoutProxies = hostnames.filterNot { forHost(it, configRegistry).hasProxies() } + return withoutProxies.isNotEmpty() && canRestartNetworkDevice() + } + // The original design changes address by restarting the device, which only works where ifconfig // exists and the connection hands out a new address afterwards. That is a property of the // machine, so ask it rather than assume either way. From 4943e3a17886ee1724d33f48796df20019980fd5 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 23:44:06 +0100 Subject: [PATCH 08/50] LiveChart: log that the newest year is detected, not a highest ID --- .../modb/app/crawlers/livechart/LivechartNewestYearDetector.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt index ca89e3c1..5a15fbaa 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt @@ -29,7 +29,7 @@ class LivechartNewestYearDetector( ): HighestIdDetector { override suspend fun detectHighestId(): Int { - log.info { "Fetching highest id for [${metaDataProviderConfig.hostname()}]." } + log.info { "Detecting newest year for [${metaDataProviderConfig.hostname()}]." } wait() From f8be2fdccbcbc32f330f7b8546614a51ee0ea550 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Tue, 27 Oct 2026 23:51:27 +0100 Subject: [PATCH 09/50] Log the aniSearch relations crawler as anisearch.com-relations --- .../app/crawlers/anisearch/AnisearchCrawler.kt | 17 +++++++++-------- .../AnisearchPaginationIdRangeSelector.kt | 3 ++- .../DefaultDownloadControlStateScheduler.kt | 5 +++-- .../MetaDataProviderConfigExtensions.kt | 16 ++++++++++++++++ 4 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt index 084ce8db..46f8056c 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchCrawler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.anisearch +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchDownloader import io.github.manamiproject.modb.app.config.AppConfig @@ -70,7 +71,7 @@ class AnisearchCrawler( metaDataProviderConfig = metaDataProviderConfig, httpClient = httpClient, ), - hostname = metaDataProviderConfig.hostname(), + hostname = metaDataProviderConfig.siteName(), ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), ): Crawler { @@ -83,26 +84,26 @@ class AnisearchCrawler( } override suspend fun start() { - log.info { "Starting crawler for [${metaDataProviderConfig.hostname()}]." } + log.info { "Starting crawler for [${metaDataProviderConfig.siteName()}]." } downloadEntriesScheduledForCurrentWeek() wait() downloadEntriesUsingPagination() - log.info { "Finished crawling data for [${metaDataProviderConfig.hostname()}]." } + log.info { "Finished crawling data for [${metaDataProviderConfig.siteName()}]." } } private suspend fun downloadEntriesScheduledForCurrentWeek() { - log.info { "Downloading [${metaDataProviderConfig.hostname()}] entries scheduled for the current week." } + log.info { "Downloading [${metaDataProviderConfig.siteName()}] entries scheduled for the current week." } val ids = downloadControlStateScheduler.findEntriesScheduledForCurrentWeek(metaDataProviderConfig) - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) startDownload(ids.toList().createShuffledList()) - log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries scheduled for the current week." } + log.info { "Finished downloading [${metaDataProviderConfig.siteName()}] entries scheduled for the current week." } } private suspend fun downloadEntriesUsingPagination() { - log.info { "Downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } + log.info { "Downloading [${metaDataProviderConfig.siteName()}] entries using pagination." } val firstPage = lastPageMemorizer.retrieveLastPage() val lastPage = lastPageDetector.detectHighestId() @@ -110,7 +111,7 @@ class AnisearchCrawler( val alreadyDownloaded = alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig).size log.info { - "[${metaDataProviderConfig.hostname()}] Skipping [$alreadyDownloaded] already downloaded " + + "[${metaDataProviderConfig.siteName()}] Skipping [$alreadyDownloaded] already downloaded " + "and [${entriesNotScheduledForCurrentWeek.size}] not scheduled for this week. " + "Continuing from page [$firstPage] of [$lastPage]." } @@ -124,7 +125,7 @@ class AnisearchCrawler( lastPageMemorizer.memorizeLastPage(currentPage) } - log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } + log.info { "Finished downloading [${metaDataProviderConfig.siteName()}] entries using pagination." } } private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt index 347828e1..ea3ab8ac 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anisearch/AnisearchPaginationIdRangeSelector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.anisearch +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.convfiles.DefaultAlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.crawlers.PaginationIdRangeSelector @@ -58,7 +59,7 @@ class AnisearchPaginationIdRangeSelector( } override suspend fun idDownloadList(page: Int): List { - log.info { "Retrieving IDs for [${metaDataProviderConfig.hostname()}] from page [$page]" } + log.info { "Retrieving IDs for [${metaDataProviderConfig.siteName()}] from page [$page]" } if (entriesNotScheduledForCurrentWeek.isEmpty()) { entriesNotScheduledForCurrentWeek.addAll(downloadControlStateScheduler.findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig)) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt index c828a7a9..c136fb2e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateScheduler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.downloadcontrolstate +import io.github.manamiproject.modb.app.extensions.siteName import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.core.config.AnimeId @@ -22,7 +23,7 @@ class DefaultDownloadControlStateScheduler( ): DownloadControlStateScheduler { override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are not scheduled to download this week for [${metaDataProviderConfig.hostname()}]." } + log.info { "Finding all anime which are not scheduled to download this week for [${metaDataProviderConfig.siteName()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isAfterCurrentWeek(it.nextDownload) } @@ -31,7 +32,7 @@ class DefaultDownloadControlStateScheduler( } override suspend fun findEntriesScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set { - log.info { "Finding all anime which are scheduled to download this week for [${metaDataProviderConfig.hostname()}]." } + log.info { "Finding all anime which are scheduled to download this week for [${metaDataProviderConfig.siteName()}]." } return downloadControlStateAccessor.allDcsEntries(metaDataProviderConfig) .filter { isCurrentWeekOrOlder(it.nextDownload) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt new file mode 100644 index 00000000..2df32b34 --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/extensions/MetaDataProviderConfigExtensions.kt @@ -0,0 +1,16 @@ +package io.github.manamiproject.modb.app.extensions + +import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig + +/** + * Name of the crawler to use in log messages. Equals the hostname, except for crawlers which share a hostname with + * another one. The names are the same as the ones used for `modb.app.disabledCrawlers`. + * @since 20.0.0 + * @receiver Config of the crawler. + * @return Name which identifies the crawler. + */ +fun MetaDataProviderConfig.siteName(): String = when (this) { + AnisearchRelationsConfig -> "${hostname()}-relations" + else -> hostname() +} From cde1e923b416ea50d84b56a4b896dadfb747417b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 01:33:21 +0100 Subject: [PATCH 10/50] AniList: use the official GraphQL API instead of the website's internal endpoint --- .../modb/anilist/AnilistConfig.kt | 3 +- .../modb/anilist/AnilistDownloader.kt | 4 +-- .../modb/anilist/AnilistHeaderCreator.kt | 29 +++++-------------- .../modb/anilist/AnilistConfigTest.kt | 2 +- .../modb/anilist/AnilistHeaderCreatorTest.kt | 26 ++++------------- .../app/crawlers/anilist/AnilistCrawler.kt | 5 +--- .../anilist/AnilistHighestIdDetector.kt | 4 +-- 7 files changed, 20 insertions(+), 53 deletions(-) diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt index cba344eb..bf65a1cb 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistConfig.kt @@ -11,7 +11,8 @@ import java.net.URI */ public object AnilistConfig : MetaDataProviderConfig { - override fun buildDataDownloadLink(id: String): URI = URI("https://${hostname()}/graphql") + // Official public GraphQL API. The anime itself is still linked on the website (hostname). + override fun buildDataDownloadLink(id: String): URI = URI("https://graphql.anilist.co") override fun hostname(): Hostname = "anilist.co" diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt index 5847847b..9ee87257 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDownloader.kt @@ -6,6 +6,7 @@ import io.github.manamiproject.modb.core.downloader.Downloader import io.github.manamiproject.modb.core.extensions.EMPTY import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank import io.github.manamiproject.modb.core.httpclient.APPLICATION_JSON +import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.httpclient.RequestBody import io.github.manamiproject.modb.core.loadResource @@ -20,7 +21,7 @@ import kotlinx.coroutines.runBlocking */ public class AnilistDownloader( private val metaDataProviderConfig: MetaDataProviderConfig = AnilistConfig, - private val httpClient: HttpClient = AnilistHttpClient(), + private val httpClient: HttpClient = DefaultHttpClient(isTestContext = metaDataProviderConfig.isTestContext()), ) : Downloader { private val requestBody: String by lazy { runBlocking { loadResource("anime_download_request.graphql") } } @@ -35,7 +36,6 @@ public class AnilistDownloader( val requestHeaders = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = metaDataProviderConfig.buildAnimeLink(id).toURL(), ) val response = httpClient.post( diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt index a3bd7cf0..f24047e4 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreator.kt @@ -1,39 +1,24 @@ package io.github.manamiproject.modb.anilist import io.github.manamiproject.modb.core.httpclient.RequestBody -import java.net.URI -import java.net.URL /** - * Creates all necessary header parameter for an anilist GraphQL request. + * Creates all necessary header parameter for a request to the official anilist GraphQL API. * @since 1.0.0 */ public object AnilistHeaderCreator { /** - * Creates all headers necessary for an anilist GraphQL request. + * Creates all headers necessary for a request to the official anilist GraphQL API. + * The public API doesn't require a session cookie or csrf token. * @since 1.0.0 - * @param requestBody The request body being sent. It's necessary for content-length parameter. - * @param referer The referer [URL]. **Default**: _https://anilist.co_ + * @param requestBody The request body being sent. * @return A [Map] of header parameter. Key is the name of the parameter itself and value is a list of values for this specific header parameter. */ - public fun createAnilistHeaders(requestBody: RequestBody, referer: URL = URI("https://anilist.co").toURL()): Map> { - val url = "https://anilist.co" - + public fun createAnilistHeaders(requestBody: RequestBody): Map> { return mapOf( - "authority" to listOf(url), - "method" to listOf("POST"), - "path" to listOf("/graphql"), - "scheme" to listOf("https"), - "accept" to listOf("*/*"), - "accept-language" to listOf("en-US;q=0.9", "en;q=0.8"), - "content-length" to listOf(requestBody.body.length.toString()), + "accept" to listOf("application/json"), "content-type" to listOf(requestBody.mediaType), - "origin" to listOf(url), - "schema" to listOf("default"), - "cookie" to listOf(AnilistDefaultTokenRepository.token.cookie), - "x-csrf-token" to listOf(AnilistDefaultTokenRepository.token.csrfToken), - "referer" to listOf(referer.toString()) ) } -} \ No newline at end of file +} diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt index 31cbe5dc..6c102066 100644 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt +++ b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistConfigTest.kt @@ -45,7 +45,7 @@ internal class AnilistConfigTest { val result = AnilistConfig.buildDataDownloadLink(id) // then - assertThat(result).isEqualTo(URI("https://anilist.co/graphql")) + assertThat(result).isEqualTo(URI("https://graphql.anilist.co")) } @Test diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt index a3422f75..a65db77a 100644 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt +++ b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHeaderCreatorTest.kt @@ -2,16 +2,13 @@ package io.github.manamiproject.modb.anilist import io.github.manamiproject.modb.core.httpclient.RequestBody import org.assertj.core.api.Assertions.assertThat -import java.net.URI import kotlin.test.Test internal class AnilistHeaderCreatorTest { @Test - fun `default referer`() { + fun `creates the headers for the official API`() { // given - AnilistDefaultTokenRepository.token = AnilistToken("cookie-value", "csrf-token-value") - val requestBody = RequestBody( mediaType = "used media type", body = "payload", @@ -23,23 +20,13 @@ internal class AnilistHeaderCreatorTest { ) // then - assertThat(result["authority"]).isEqualTo(listOf("https://anilist.co")) - assertThat(result["method"]).isEqualTo(listOf("POST")) - assertThat(result["path"]).isEqualTo(listOf("/graphql")) - assertThat(result["scheme"]).isEqualTo(listOf("https")) - assertThat(result["accept"]).isEqualTo(listOf("*/*")) - assertThat(result["accept-language"]).isEqualTo(listOf("en-US;q=0.9", "en;q=0.8")) - assertThat(result["content-length"]).isEqualTo(listOf("7")) + assertThat(result).containsOnlyKeys("accept", "content-type") + assertThat(result["accept"]).isEqualTo(listOf("application/json")) assertThat(result["content-type"]).isEqualTo(listOf("used media type")) - assertThat(result["cookie"]).isEqualTo(listOf("cookie-value")) - assertThat(result["origin"]).isEqualTo(listOf("https://anilist.co")) - assertThat(result["schema"]).isEqualTo(listOf("default")) - assertThat(result["x-csrf-token"]).isEqualTo(listOf("csrf-token-value")) - assertThat(result["referer"]).isEqualTo(listOf("https://anilist.co")) } @Test - fun `override referer`() { + fun `doesn't send session cookie or csrf token`() { // given val requestBody = RequestBody( mediaType = "used media type", @@ -49,10 +36,9 @@ internal class AnilistHeaderCreatorTest { // when val result = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = URI("https://anilist.co/anime/1535").toURL(), ) // then - assertThat(result["referer"]).isEqualTo(listOf("https://anilist.co/anime/1535")) + assertThat(result).doesNotContainKeys("cookie", "x-csrf-token") } -} \ No newline at end of file +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt index c004d759..1a1ad75b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistCrawler.kt @@ -2,7 +2,6 @@ package io.github.manamiproject.modb.app.crawlers.anilist import io.github.manamiproject.modb.anilist.AnilistConfig import io.github.manamiproject.modb.anilist.AnilistDownloader -import io.github.manamiproject.modb.anilist.AnilistHttpClient import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.crawlers.Crawler @@ -45,10 +44,8 @@ class AnilistCrawler( ), private val downloader: Downloader = FaultTolerantDownloader( downloader = AnilistDownloader( - httpClient = SuspendableHttpClient( - httpClient = AnilistHttpClient.instance, + httpClient = SuspendableHttpClient(), ), - ), hostname = metaDataProviderConfig.hostname(), ), ): Crawler { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt index ba500b0f..39ca216b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anilist/AnilistHighestIdDetector.kt @@ -2,7 +2,6 @@ package io.github.manamiproject.modb.app.crawlers.anilist import io.github.manamiproject.modb.anilist.AnilistConfig import io.github.manamiproject.modb.anilist.AnilistHeaderCreator -import io.github.manamiproject.modb.anilist.AnilistHttpClient import io.github.manamiproject.modb.app.crawlers.HighestIdDetector import io.github.manamiproject.modb.app.extensions.checkedBody import io.github.manamiproject.modb.app.network.SuspendableHttpClient @@ -22,7 +21,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate */ class AnilistHighestIdDetector( private val metaDataProviderConfig: MetaDataProviderConfig = AnilistConfig, - private val httpClient: HttpClient = SuspendableHttpClient(httpClient = AnilistHttpClient.instance), + private val httpClient: HttpClient = SuspendableHttpClient(), ): HighestIdDetector { override suspend fun detectHighestId(): Int { @@ -39,7 +38,6 @@ class AnilistHighestIdDetector( url = requestUrl.toURL(), headers = AnilistHeaderCreator.createAnilistHeaders( requestBody = requestBody, - referer = requestUrl.toURL(), ), requestBody = requestBody, ).checkedBody(this::class) From 847d7a371e038f76fc10516e11790756364f8674 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 01:56:58 +0100 Subject: [PATCH 11/50] Add config.toml template --- config.example.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 config.example.toml diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 00000000..594e1d22 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,13 @@ +# Copy to config.toml and fill in the values. All directories must exist. + +[modb.app] +downloadsDirectory = "" +outputDirectory = "" +downloadControlStateDirectory = "" +# Crawlers to skip. +disabledCrawlers = ["anidb.net"] + +[modb.anidb] +# Client registered for the AniDB HTTP API. +client = "" +clientver = "" From d272eab5aaafc2ec81b6dced3de92ea7ef3046dd Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 23 Oct 2026 03:31:11 +0200 Subject: [PATCH 12/50] kommand: recognise missing commands on newer JDKs, skip sudo password test without password prompt --- .../manamiproject/kommand/JavaProcessBuilder.kt | 3 ++- .../kommand/JavaProcessBuilderTest.kt | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt index b2699028..6b80c8b8 100644 --- a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt +++ b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt @@ -98,7 +98,8 @@ public class JavaProcessBuilder( } when { - e.cause?.message == "error=2, No such file or directory" -> throw CommandNotFoundException(commandName) + // The wording depends on the JDK, e.g. "error=2, No such file or directory" or "Exec failed, error: 2 (No such file or directory)" + e.cause?.message?.contains("No such file or directory") == true -> throw CommandNotFoundException(commandName) e.cause?.message?.contains("permission denied", false) == true -> throw PermissionDeniedException(commandName) else -> throw e } diff --git a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt index 3edc0682..29ba8562 100644 --- a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt +++ b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt @@ -5,6 +5,7 @@ import io.github.manamiproject.kommand.exceptions.IncorrectPasswordAttemptExcept import io.github.manamiproject.kommand.exceptions.UnexpectedUsageOfSudo import io.github.manamiproject.kommand.exceptions.UnsupportedOperatingSystemException import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -186,6 +187,8 @@ internal class JavaProcessBuilderTest { @Test fun `throws exception if command with sudo is given an incorrect password`() { + assumeTrue(sudoRequiresPassword(), "sudo doesn't ask for a password on this system, so an incorrect password cannot be detected.") + // given val javaProcessBuilder = JavaProcessBuilder { useSudo = true @@ -219,4 +222,16 @@ internal class JavaProcessBuilderTest { assertThat(result===previous).isTrue() } } -} \ No newline at end of file +} + +private fun sudoRequiresPassword(): Boolean { + return try { + // -k ignores cached credentials, -n fails instead of asking if a password is required + ProcessBuilder("sudo", "-k", "-n", "true") + .redirectErrorStream(true) + .start() + .waitFor() != 0 + } catch (e: java.io.IOException) { + false + } +} From 031b10532ceea7c161afb5036e9349d74340622c Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 02:47:33 +0100 Subject: [PATCH 13/50] Refactor sitesChangingAddress to use buildSet Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../main/kotlin/io/github/manamiproject/modb/app/App.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 5ccc2faa..68884cf3 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -42,7 +42,14 @@ fun main() = runCoroutine { val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() // Only the crawlers of these sites change the outgoing address by restarting the network device. - val sitesChangingAddress = setOf(AnidbConfig.hostname(), AnisearchConfig.hostname()) - disabled + val sitesChangingAddress = buildSet { + if (AnidbConfig.hostname() !in disabled) { + add(AnidbConfig.hostname()) + } + if (AnisearchConfig.hostname() !in disabled || "anisearch.com-relations" !in disabled) { + add(AnisearchConfig.hostname()) + } + } when { isRoot() -> log.info("Running as root. No sudo password needed.") From 89a193e04b496af817c41f0062bbae6908e510c2 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 02:58:36 +0100 Subject: [PATCH 14/50] Don't prefix ifconfig with sudo when sudo is disabled, ask for the sudo password if either aniSearch crawler is enabled --- .../io/github/manamiproject/modb/app/App.kt | 6 ++- .../modb/app/network/ifconfig.kt | 3 +- .../modb/app/network/IfconfigKtTest.kt | 54 ++++++++++++++++++- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 68884cf3..ee4215f9 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -42,11 +42,12 @@ fun main() = runCoroutine { val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() // Only the crawlers of these sites change the outgoing address by restarting the network device. + // Both aniSearch crawlers use the hostname of anisearch.com, but can be disabled independently. val sitesChangingAddress = buildSet { if (AnidbConfig.hostname() !in disabled) { add(AnidbConfig.hostname()) } - if (AnisearchConfig.hostname() !in disabled || "anisearch.com-relations" !in disabled) { + if (AnisearchConfig.hostname() !in disabled || ANISEARCH_RELATIONS_CRAWLER !in disabled) { add(AnisearchConfig.hostname()) } } @@ -68,7 +69,7 @@ fun main() = runCoroutine { crawl("anime-planet.com", disabled, crawlers) { AnimePlanetCrawler.instance.start() } crawl("animenewsnetwork.com", disabled, crawlers) { AnimenewsnetworkCrawler.instance.start() } crawl("anisearch.com", disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchConfig).start() } - crawl("anisearch.com-relations", disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchRelationsConfig).start() } + crawl(ANISEARCH_RELATIONS_CRAWLER, disabled, crawlers) { AnisearchCrawler(metaDataProviderConfig = AnisearchRelationsConfig).start() } crawl("kitsu.app", disabled, crawlers) { KitsuCrawler.instance.start() } crawl("livechart.me", disabled, crawlers) { LivechartCrawler.instance.start() } crawl("myanimelist.net", disabled, crawlers) { MyanimelistCrawler.instance.start() } @@ -147,6 +148,7 @@ private fun isValidSudoPassword(password: String): Boolean = try { } private const val MAX_SUDO_ATTEMPTS = 3 +private const val ANISEARCH_RELATIONS_CRAWLER = "anisearch.com-relations" @KoverIgnore private fun passwordPrompt(): String { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt index e4ecebad..bfbb53db 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ifconfig.kt @@ -59,7 +59,8 @@ internal fun ifconfig(device: Device, vararg options: String, config: IfconfigCo val currentConfig = IfconfigConfig().apply(config) val commandBuilder = mutableListOf() - if (options.contains("up") || options.contains("down")) { + // Only with sudo enabled. Running as root, the executor rejects a command starting with sudo. + if ((options.contains("up") || options.contains("down")) && currentConfig.commandExecutor.config.useSudo) { commandBuilder.add("sudo") } commandBuilder.add("ifconfig") diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt index 8e07f426..642b8907 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/IfconfigKtTest.kt @@ -150,7 +150,7 @@ internal class IfconfigKtTest { // given val receivedCommand = mutableListOf() val testCommandExecutor = object : CommandExecutor { - override var config: CommandLineConfig = CommandLineConfig() + override var config: CommandLineConfig = CommandLineConfig(useSudo = true) override fun executeCmd(command: List): String { receivedCommand.addAll(command) return loadTestResource("network/IfconfigKtTest/device_up.txt") @@ -176,7 +176,7 @@ internal class IfconfigKtTest { // given val receivedCommand = mutableListOf() val testCommandExecutor = object : CommandExecutor { - override var config: CommandLineConfig = CommandLineConfig() + override var config: CommandLineConfig = CommandLineConfig(useSudo = true) override fun executeCmd(command: List): String { receivedCommand.addAll(command) return loadTestResource("network/IfconfigKtTest/device_down.txt") @@ -197,6 +197,56 @@ internal class IfconfigKtTest { ) } + @Test + fun `up a specific device without sudo if sudo is disabled`() { + // given + val receivedCommand = mutableListOf() + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig(useSudo = false) + override fun executeCmd(command: List): String { + receivedCommand.addAll(command) + return loadTestResource("network/IfconfigKtTest/device_up.txt") + } + } + + // when + ifconfig("en1", "up") { + commandExecutor = testCommandExecutor + } + + // then + assertThat(receivedCommand).containsExactly( + "ifconfig", + "en1", + "up", + ) + } + + @Test + fun `down a specific device without sudo if sudo is disabled`() { + // given + val receivedCommand = mutableListOf() + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig(useSudo = false) + override fun executeCmd(command: List): String { + receivedCommand.addAll(command) + return loadTestResource("network/IfconfigKtTest/device_down.txt") + } + } + + // when + ifconfig("en1", "down") { + commandExecutor = testCommandExecutor + } + + // then + assertThat(receivedCommand).containsExactly( + "ifconfig", + "en1", + "down", + ) + } + @Test fun `calls ifconfig without parameters if asked for password`() { // given From ad113a9da4067a1c0b2c15c221d4e787f40eac9d Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 03:44:01 +0100 Subject: [PATCH 15/50] AniList: remove the unused website token handling --- .../anilist/AnilistDefaultTokenRepository.kt | 21 - .../anilist/AnilistDefaultTokenRetriever.kt | 83 ---- .../AnilistDefaultTokenRetrieverConfig.kt | 13 - .../modb/anilist/AnilistHttpClient.kt | 67 --- .../modb/anilist/AnilistToken.kt | 12 - .../modb/anilist/AnilistTokenRepository.kt | 16 - .../modb/anilist/AnilistTokenRetriever.kt | 14 - .../AnilistDefaultTokenRepositoryTest.kt | 65 --- .../AnilistDefaultTokenRetrieverConfigTest.kt | 50 --- .../AnilistDefaultTokenRetrieverTest.kt | 205 --------- .../modb/anilist/AnilistHttpClientTest.kt | 388 ------------------ .../page_containing_token.html | 66 --- 12 files changed, 1000 deletions(-) delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt delete mode 100644 anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt delete mode 100644 anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt delete mode 100644 anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt delete mode 100644 anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt delete mode 100644 anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt delete mode 100644 anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt deleted file mode 100644 index 0a1a7a04..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepository.kt +++ /dev/null @@ -1,21 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank - -/** - * Central place to store the current anilist token so that the same token is always being used. - * @since 1.0.0 - */ -public object AnilistDefaultTokenRepository : AnilistTokenRepository { - - private var currentToken = AnilistToken(EMPTY, EMPTY) - - override var token: AnilistToken - get() = currentToken - set(value) { - require(value.cookie.neitherNullNorBlank()) { "Cookie must not blank" } - require(value.csrfToken.neitherNullNorBlank()) { "CSRF token must not blank" } - currentToken = value - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt deleted file mode 100644 index ffebed23..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetriever.kt +++ /dev/null @@ -1,83 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_CPU -import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK -import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank -import io.github.manamiproject.modb.core.extensions.remove -import io.github.manamiproject.modb.core.extractor.DataExtractor -import io.github.manamiproject.modb.core.extractor.XmlDataExtractor -import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.logging.LoggerDelegate -import kotlinx.coroutines.withContext - -private const val CSRF_TOKEN_PREFIX = "window.al_token" - -/** - * Retrieves a valid token. - * @since 1.0.0 - * @property metaDataProviderConfig Configuration for retrieving the token. - * @property httpClient To download the site from which the token will be extracted - * @property extractor Extractor which retrieves the data from raw data. - */ -public class AnilistDefaultTokenRetriever( - private val metaDataProviderConfig: MetaDataProviderConfig = AnilistDefaultTokenRetrieverConfig, - private val httpClient: HttpClient = DefaultHttpClient(isTestContext=metaDataProviderConfig.isTestContext()), - private val extractor: DataExtractor = XmlDataExtractor, -): AnilistTokenRetriever { - - override suspend fun retrieveToken(): AnilistToken = withContext(LIMITED_NETWORK) { - log.info { "Fetching token for anilist." } - - return@withContext httpClient.get( - url = metaDataProviderConfig.buildDataDownloadLink().toURL(), - ).use { response -> - val cookie = extractCookie(response) - val csrfToken = extractCsrfToken(response) - - AnilistToken( - cookie = cookie, - csrfToken = csrfToken, - ) - } - } - - private fun extractCookie(response: HttpResponse): String { - return response.headers["set-cookie"] - ?.map { it.split(';') } - ?.map { it.first() } - ?.sorted() - ?.joinToString("; ") ?: throw IllegalStateException("Unable to extract cookie.") - } - - private suspend fun extractCsrfToken(response: HttpResponse): String = withContext(LIMITED_CPU) { - val responseBody = response.bodyAsString() - require(responseBody.neitherNullNorBlank()) { "Response body must not be empty" } - - val data = extractor.extract(responseBody, selection = mapOf( - "script" to "//script[contains(node(), $CSRF_TOKEN_PREFIX)]/node()" - )) - - return@withContext if (data.notFound("script")) { - throw IllegalStateException("Unable to extract CSRF token.") - } else { - data.listNotNull("script") - .first { it.startsWith(CSRF_TOKEN_PREFIX) } - .remove("$CSRF_TOKEN_PREFIX = \"") - .remove("\";") - .trim() - } - } - - public companion object { - private val log by LoggerDelegate() - - /** - * Singleton of [AnilistDefaultTokenRetriever] - * @since 6.1.0 - */ - public val instance: AnilistDefaultTokenRetriever by lazy { AnilistDefaultTokenRetriever() } - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt deleted file mode 100644 index f9fa548e..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfig.kt +++ /dev/null @@ -1,13 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import java.net.URI - -/** - * Configuration for retrieving the CSRF token - * @since 1.0.0 - */ -public object AnilistDefaultTokenRetrieverConfig : MetaDataProviderConfig by AnilistConfig { - - override fun buildDataDownloadLink(id: String): URI = URI("https://${hostname()}") -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt deleted file mode 100644 index f8584851..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClient.kt +++ /dev/null @@ -1,67 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.httpclient.RequestBody -import io.github.manamiproject.modb.core.logging.LoggerDelegate -import java.net.URL - -/** - * Anilist specific implementation of a [HttpClient] which handles the renewal of the token under the hood. - * @since 5.1.1 - * @property delegate HttpClient which is used under the hood. - * @property anilistTokenRetriever Fetches a new token for the CSRF. - * @property anilistTokenRepository Keeps the current CSRF token in-memory. - */ -public class AnilistHttpClient( - private val delegate: HttpClient = DefaultHttpClient(), - private val anilistTokenRetriever: AnilistTokenRetriever = AnilistDefaultTokenRetriever.instance, - private val anilistTokenRepository: AnilistTokenRepository = AnilistDefaultTokenRepository, -): HttpClient by delegate { - - override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { - val initialResponse = delegate.post(url, requestBody, headers) - - if (initialResponse.code != 403) { - return initialResponse - } - - initialResponse.close() - return delegate.post(url, requestBody, renewTokenInHeaders(headers)) - } - - override suspend fun get(url: URL, headers: Map>): HttpResponse { - val initialResponse = delegate.get(url, headers) - - if (initialResponse.code != 403) { - return initialResponse - } - - initialResponse.close() - return delegate.get(url, renewTokenInHeaders(headers)) - } - - private suspend fun renewTokenInHeaders(headers: Map>): Map> { - log.warn { "Anilist responds with 403. Refreshing token." } - anilistTokenRepository.token = anilistTokenRetriever.retrieveToken() - val modifiedHeaders = HashMap(headers).apply { - put("cookie", listOf(AnilistDefaultTokenRepository.token.cookie)) - put("x-csrf-token", listOf(AnilistDefaultTokenRepository.token.csrfToken)) - } - log.info { "Token has been renewed" } - - return modifiedHeaders - } - - - public companion object { - private val log by LoggerDelegate() - - /** - * Singleton of [AnilistHttpClient] - * @since 1.0.0 - */ - public val instance: AnilistHttpClient by lazy { AnilistHttpClient() } - } -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt deleted file mode 100644 index 8bd83a8d..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistToken.kt +++ /dev/null @@ -1,12 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Contains the CSRF token and the corresponding cookie value. - * @since 1.0.0 - * @property cookie Value for cookies header parameter - * @property csrfToken Value for CSRF token header parameter - */ -public data class AnilistToken ( - val cookie: String, - val csrfToken: String, -) \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt deleted file mode 100644 index 651ecadc..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRepository.kt +++ /dev/null @@ -1,16 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Stores the current anilist token. - * @since 2.0.0 - */ -public interface AnilistTokenRepository { - - /** - * Currently valid token or a token with empty fields after initialization. - * The token is only accepted if both cookie and csrf field are not blank. - * @since 1.0.0 - * @throws IllegalArgumentException if the given token contains blank fields - */ - public var token: AnilistToken -} \ No newline at end of file diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt deleted file mode 100644 index b704013b..00000000 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistTokenRetriever.kt +++ /dev/null @@ -1,14 +0,0 @@ -package io.github.manamiproject.modb.anilist - -/** - * Retrieves the [AnilistToken] which is needed to query the GraphQL endpoints. - * @since 1.0.0 - */ -public interface AnilistTokenRetriever { - - /** - * @since 5.0.0 - * @return Valid [AnilistToken] which is needed to download data - */ - public suspend fun retrieveToken(): AnilistToken -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt deleted file mode 100644 index 31b5e27e..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRepositoryTest.kt +++ /dev/null @@ -1,65 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import org.assertj.core.api.Assertions.assertThat -import kotlin.test.Test -import org.junit.jupiter.api.assertThrows - -internal class AnilistDefaultTokenRepositoryTest { - - @Test - fun `successfully set token`() { - // given - val token = AnilistToken("my-cookie-value", "my-csrf-token-value") - - // when - AnilistDefaultTokenRepository.token = token - - // then - assertThat(AnilistDefaultTokenRepository.token).isEqualTo(token) - } - - @Test - fun `throws exception if cookie is blank`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken(" ", "test") - } - - // then - assertThat(result).hasMessage("Cookie must not blank") - } - - @Test - fun `throws exception if cookie is empty`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken(EMPTY, "test") - } - - // then - assertThat(result).hasMessage("Cookie must not blank") - } - - @Test - fun `throws exception if csrf token is blank`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken("test", " ") - } - - // then - assertThat(result).hasMessage("CSRF token must not blank") - } - - @Test - fun `throws exception if csrf token is empty`() { - // when - val result = assertThrows { - AnilistDefaultTokenRepository.token = AnilistToken("test", EMPTY) - } - - // then - assertThat(result).hasMessage("CSRF token must not blank") - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt deleted file mode 100644 index e1f20df4..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverConfigTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import org.assertj.core.api.Assertions.assertThat -import java.net.URI -import kotlin.test.Test - -internal class AnilistDefaultTokenRetrieverConfigTest { - - @Test - fun `hostname must be correct`() { - // when - val result = AnilistDefaultTokenRetrieverConfig.hostname() - - // then - assertThat(result).isEqualTo("anilist.co") - } - - @Test - fun `build anime link correctly`() { - // given - val id = "4563" - - // when - val result = AnilistDefaultTokenRetrieverConfig.buildAnimeLink(id) - - // then - assertThat(result).isEqualTo(URI("https://anilist.co/anime/4563")) - } - - @Test - fun `build data download link correctly`() { - // given - val type = "music" - - // when - val result = AnilistDefaultTokenRetrieverConfig.buildDataDownloadLink(type) - - // then - assertThat(result).isEqualTo(URI("https://anilist.co")) - } - - @Test - fun `file suffix must be html`() { - // when - val result = AnilistDefaultTokenRetrieverConfig.fileSuffix() - - // then - assertThat(result).isEqualTo("json") - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt deleted file mode 100644 index fb861dad..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistDefaultTokenRetrieverTest.kt +++ /dev/null @@ -1,205 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.client.WireMock.* -import io.github.manamiproject.modb.core.config.FileSuffix -import io.github.manamiproject.modb.core.config.Hostname -import io.github.manamiproject.modb.core.config.MetaDataProviderConfig -import io.github.manamiproject.modb.core.httpclient.APPLICATION_JSON -import io.github.manamiproject.modb.test.MockServerTestCase -import io.github.manamiproject.modb.test.WireMockServerCreator -import io.github.manamiproject.modb.test.exceptionExpected -import io.github.manamiproject.modb.test.loadTestResource -import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Nested -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ValueSource -import java.net.URI -import kotlin.test.Test - -internal class AnilistDefaultTokenRetrieverTest : MockServerTestCase by WireMockServerCreator() { - - @ParameterizedTest - @ValueSource(strings = [ - "", - " ", - "\u00A0", - "\u202F", - "\u200A", - "\u205F", - "\u2000", - "\u2001", - "\u2002", - "\u2003", - "\u2004", - "\u2005", - "\u2006", - "\u2007", - "\u2008", - "\u2009", - "\uFEFF", - "\u180E", - "\u2060", - "\u200D", - "\u0090", - "\u200C", - "\u200B", - "\u00AD", - "\u000C", - "\u2028", - "\r", - "\n", - "\t", - ]) - fun `throws exception if the response body is blank`(value: String) { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly" - ) - .withStatus(200) - .withBody(value) - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Response body must not be empty") - } - - @Test - fun `throws exception if the csrf token cannot be retrieved`() { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly") - .withStatus(200) - .withBody("response body without token") - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Unable to extract CSRF token.") - } - - @Test - fun `throw exception if the cookie cannot be retrieved`() { - // given - val testAnilistConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - val responseBody = loadTestResource("AnilistDefaultTokenRetrieverTest/page_containing_token.html") - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withStatus(200) - .withBody(responseBody) - ) - ) - - // when - val result = exceptionExpected { - anilistTokenRetriever.retrieveToken() - } - - // then - assertThat(result).hasMessage("Unable to extract cookie.") - } - - @Test - fun `correctly retrieve token`() { - runTest { - // given - val testAnilistConfig = object : MetaDataProviderConfig by TestMetaDataProviderConfig { - override fun hostname(): Hostname = "localhost" - override fun buildDataDownloadLink(id: String): URI = URI("http://${hostname()}:$port/$id") - override fun fileSuffix(): FileSuffix = AnilistConfig.fileSuffix() - } - - val anilistTokenRetriever = AnilistDefaultTokenRetriever(testAnilistConfig) - - val responseBody = loadTestResource("AnilistDefaultTokenRetrieverTest/page_containing_token.html") - - serverInstance.stubFor( - get(urlPathEqualTo("/")).willReturn( - aResponse() - .withHeader("Content-Type", APPLICATION_JSON) - .withHeader( - "set-cookie", - "__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; expires=Fri, 21-Feb-20 20:56:37 GMT; path=/; domain=.anilist.co; HttpOnly; SameSite=Lax; Secure", - "laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL; expires=Thu, 23-Jan-2020 08:56:37 GMT; Max-Age=43200; path=/; httponly" - ) - .withStatus(200) - .withBody(responseBody) - ) - ) - - // when - val result = anilistTokenRetriever.retrieveToken() - - // then - assertThat(result.cookie).isEqualTo("__cfduid=db93afbdcce117dd877b809ce8b6dde941579726597; laravel_session=NOz33Vu7KGVZK4TZqSES3lmv14JmKbe9IrHN4LnL") - assertThat(result.csrfToken).isEqualTo("IAasRzCsdYp2b5QWQEWtMzSvDzf8UboK0GiH907Y") - } - } - - @Nested - inner class CompanionObjectTests { - - @Test - fun `instance property always returns same instance`() { - // given - val previous = AnilistDefaultTokenRetriever.instance - - // when - val result = AnilistDefaultTokenRetriever.instance - - // then - assertThat(result).isExactlyInstanceOf(AnilistDefaultTokenRetriever::class.java) - assertThat(result === previous).isTrue() - } - } -} \ No newline at end of file diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt deleted file mode 100644 index 83375b67..00000000 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistHttpClientTest.kt +++ /dev/null @@ -1,388 +0,0 @@ -package io.github.manamiproject.modb.anilist - -import io.github.manamiproject.modb.core.extensions.EMPTY -import io.github.manamiproject.modb.core.httpclient.HttpClient -import io.github.manamiproject.modb.core.httpclient.HttpResponse -import io.github.manamiproject.modb.core.httpclient.RequestBody -import io.github.manamiproject.modb.test.shouldNotBeInvoked -import io.github.manamiproject.modb.test.tempDirectory -import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Nested -import java.net.URI -import java.net.URL -import kotlin.test.Test - -internal class AnilistHttpClientTest { - - @Nested - inner class GetTests { - - @Test - fun `successfully loads an entry and retrieves a token upon first call`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - } - } - - var tokenHasBeenSet = false - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { tokenHasBeenSet = true } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(tokenHasBeenSet).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - - @Test - fun `updates token if http response code is 403`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "updated-cookie-value", - csrfToken = "updated-csrf-value", - ) - } - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - private var currentToken = AnilistToken("initial-cookie-value", "initial-csrf-token") - - override var token: AnilistToken - get() = currentToken - set(value) { currentToken = value } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - assertThat(testAnilistDefaultTokenRepository.token.cookie).isEqualTo("updated-cookie-value") - assertThat(testAnilistDefaultTokenRepository.token.csrfToken).isEqualTo("updated-csrf-value") - } - } - - @Test - fun `doesn't trigger token renewal if status code is 200`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun get(url: URL, headers: Map>): HttpResponse { - delegateCallTimes++ - return HttpResponse( - code = 200, - body = "success", - ) - } - } - - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken = shouldNotBeInvoked() - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { shouldNotBeInvoked() } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.get(url = URI("http://localhost").toURL()) - - // then - assertThat(delegateCallTimes).isEqualTo(1) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - } - - @Nested - inner class PostTests { - - @Test - fun `successfully loads an entry and retrieves a token upon first call`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - } - } - - var tokenHasBeenSet = false - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { tokenHasBeenSet = true } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(tokenHasBeenSet).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - - @Test - fun `updates token if http response code is 403`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return when (delegateCallTimes) { - 1 -> HttpResponse( - code = 403, - body = "token outdated", - ) - else -> HttpResponse( - code = 200, - body = "success", - ) - } - } - } - - var tokenHasBeenRetrieved = false - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken { - tokenHasBeenRetrieved = true - return AnilistToken( - cookie = "updated-cookie-value", - csrfToken = "updated-csrf-value", - ) - } - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - private var currentToken = AnilistToken("initial-cookie-value", "initial-csrf-token") - - override var token: AnilistToken - get() = currentToken - set(value) { currentToken = value } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(tokenHasBeenRetrieved).isTrue() - assertThat(delegateCallTimes).isEqualTo(2) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - assertThat(testAnilistDefaultTokenRepository.token.cookie).isEqualTo("updated-cookie-value") - assertThat(testAnilistDefaultTokenRepository.token.csrfToken).isEqualTo("updated-csrf-value") - } - } - - @Test - fun `doesn't trigger token renewal if status code is 200`() { - runTest { - // given - var delegateCallTimes = 0 - val testHttpClient = object: HttpClient by TestHttpClient { - override suspend fun post( - url: URL, - requestBody: RequestBody, - headers: Map> - ): HttpResponse { - delegateCallTimes++ - return HttpResponse( - code = 200, - body = "success", - ) - } - } - - val testAnilistDefaultTokenRetriever = object : AnilistTokenRetriever { - override suspend fun retrieveToken(): AnilistToken = shouldNotBeInvoked() - } - - val testAnilistDefaultTokenRepository = object: AnilistTokenRepository { - override var token: AnilistToken - get() = AnilistToken( - cookie = "cookie-value", - csrfToken = "csrf-value", - ) - set(_) { shouldNotBeInvoked() } - - } - - val client = AnilistHttpClient( - delegate = testHttpClient, - anilistTokenRetriever = testAnilistDefaultTokenRetriever, - anilistTokenRepository = testAnilistDefaultTokenRepository, - ) - - // when - val result = client.post(url = URI("http://localhost").toURL(), requestBody = RequestBody(EMPTY, EMPTY)) - - // then - assertThat(delegateCallTimes).isEqualTo(1) - assertThat(result.bodyAsString()).isEqualTo("success") - assertThat(result.code).isEqualTo(200) - } - } - } - - @Nested - inner class CompanionObjectTests { - - @Test - fun `instance property always returns same instance`() { - tempDirectory { - // given - val previous = AnilistHttpClient.instance - - // when - val result = AnilistHttpClient.instance - - // then - assertThat(result).isExactlyInstanceOf(AnilistHttpClient::class.java) - assertThat(result === previous).isTrue() - } - } - } -} diff --git a/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html b/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html deleted file mode 100644 index 1d3fd749..00000000 --- a/anilist/src/test/resources/AnilistDefaultTokenRetrieverTest/page_containing_token.html +++ /dev/null @@ -1,66 +0,0 @@ - - -AniList - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - \ No newline at end of file From 0f419aee519b6ee9c096d2dbd411b55699c17b9d Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Wed, 28 Oct 2026 04:03:43 +0100 Subject: [PATCH 16/50] Fix LinuxNetworkControllerTest for sudo only with password --- .../app/network/LinuxNetworkControllerTest.kt | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt index 63e53958..d01ea94a 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/LinuxNetworkControllerTest.kt @@ -106,6 +106,7 @@ internal class LinuxNetworkControllerTest { appConfig = testAppConfig, kotlinCommandExecutor = testCommandExecutor, ) + linuxNetworkController.sudoPasswordValue = "password" // when val result = linuxNetworkController.restartAsync().await() @@ -127,6 +128,70 @@ internal class LinuxNetworkControllerTest { } } + @Test + fun `restarts the network controller without sudo if no sudo password is set`() { + runTest { + // given + var clock = Clock.fixed(Instant.parse("2021-01-31T16:02:42.00Z"), UTC) + + val testAppConfig = object: Config by TestAppConfig { + override fun isTestContext(): Boolean = true + override fun clock(): Clock = clock + } + + val invocations = mutableListOf>() + var numberOfChecks = 0 + val testCommandExecutor = object : CommandExecutor { + override var config: CommandLineConfig = CommandLineConfig() + override fun executeCmd(command: List): String { + invocations.add(command) + clock = Clock.fixed(clock.instant().plus(1, ChronoUnit.MINUTES), UTC) + + return when { + command.containsExactlyInTheSameOrder(listOf("ifconfig")) -> loadTestResource("network/LinuxNetworkControllerTest/all_devices.txt") + command.containsExactlyInTheSameOrder(listOf("ifconfig", "en1")) -> { + numberOfChecks++ + + when { + numberOfChecks < 3 -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + numberOfChecks == 3 -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + numberOfChecks in 4..6 -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + numberOfChecks == 7 -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + else -> shouldNotBeInvoked() + } + } + command.any { it.contains("up") } -> loadTestResource("network/LinuxNetworkControllerTest/device_down.txt") + command.any { it.contains("down") } -> loadTestResource("network/LinuxNetworkControllerTest/device_up.txt") + else -> shouldNotBeInvoked() + } + } + } + + val linuxNetworkController = LinuxNetworkController( + appConfig = testAppConfig, + kotlinCommandExecutor = testCommandExecutor, + ) + + // when + val result = linuxNetworkController.restartAsync().await() + + // then + assertThat(result).isTrue() + assertThat(invocations).containsExactly( + listOf("ifconfig"), + listOf("ifconfig", "en1", "down"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1", "up"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + listOf("ifconfig", "en1"), + ) + } + } + @Test fun `throws exception if timeout hits while waiting for device changing status`() { runTest { From c5bf6d15756607302f1a0086505b7199b206b010 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 15:48:22 +0100 Subject: [PATCH 17/50] Make the number of retries configurable --- .../modb/anidb/AnidbApiDownloader.kt | 18 ++++++--- .../modb/anidb/AnidbApiDownloaderTest.kt | 30 +++++++++++++++ .../modb/core/httpclient/DefaultHttpClient.kt | 26 ++++++++++++- .../httpclient/DefaultHttpClientKtTest.kt | 38 +++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt index 7b0c62b0..d26683aa 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt @@ -4,6 +4,7 @@ import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.DefaultConfigRegistry import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.config.StringPropertyDelegate import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK import io.github.manamiproject.modb.core.downloader.Downloader @@ -28,7 +29,7 @@ import java.util.zip.GZIPInputStream * configuration rather than constants. * @since 1.0.0 * @property metaDataProviderConfig Configuration for downloading data. - * @property configRegistry Source of the registered client name and version. + * @property configRegistry Source of the registered client name and version and of `modb.anidb.banRetries`, the number of retries while banned. **Default:** `5` * @property httpClient To actually download the anime data. */ public class AnidbApiDownloader( @@ -47,6 +48,13 @@ public class AnidbApiDownloader( namespace = CONFIG_NAMESPACE, ) + private val banRetries: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = CONFIG_NAMESPACE, + default = DEFAULT_BAN_RETRIES, + validator = { it >= 0 }, + ) + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = withContext(LIMITED_NETWORK) { log.debug { "Downloading [anidbId=$id]" } @@ -58,15 +66,15 @@ public class AnidbApiDownloader( } catch (e: AnidbBannedException) { attempt++ - if (attempt > MAX_BAN_RETRIES) { - log.error { "Still banned by the API after [$MAX_BAN_RETRIES] retries. Giving up on [anidbId=$id]." } + if (attempt > banRetries) { + log.error { "Still banned by the API after [$banRetries] retries. Giving up on [anidbId=$id]." } throw e } // Same waiting time as the retries of the http client. The ban is answered with // http 200, so the http client itself doesn't see it as a reason to retry. val waitTime = (random(MIN_BAN_WAIT_IN_MILLISECONDS, MAX_BAN_WAIT_IN_MILLISECONDS) * attempt).toDuration(MILLISECONDS) - log.warn { "Banned by the API. Retry [$attempt/$MAX_BAN_RETRIES] for [anidbId=$id] in [$waitTime]." } + log.warn { "Banned by the API. Retry [$attempt/$banRetries] for [anidbId=$id] in [$waitTime]." } excludeFromTestContext(metaDataProviderConfig) { delay(waitTime) @@ -126,7 +134,7 @@ public class AnidbApiDownloader( private const val PROTOCOL_VERSION = 1 private const val BANNED_MESSAGE = "banned" - private const val MAX_BAN_RETRIES = 5 + private const val DEFAULT_BAN_RETRIES = 5 private const val MIN_BAN_WAIT_IN_MILLISECONDS = 60000 private const val MAX_BAN_WAIT_IN_MILLISECONDS = 90000 private const val EMPTY_RESPONSE = "" diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt index 212245a1..1068c4de 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloaderTest.kt @@ -72,6 +72,35 @@ internal class AnidbApiDownloaderTest { assertThat(invocations).isEqualTo(6) } + @Test + fun `number of retries while banned can be set via banRetries`() { + // given + var invocations = 0 + val testHttpClient = respondingWith { + invocations++ + """banned""" + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun string(key: String): String = "test" + override fun int(key: String): Int? = if (key == "modb.anidb.banRetries") 2 else null + } + + val downloader = AnidbApiDownloader( + metaDataProviderConfig = testAnidbConfig, + configRegistry = testConfigRegistry, + httpClient = testHttpClient, + ) + + // when + exceptionExpected { + downloader.download("1") { shouldNotBeInvoked() } + } + + // then + assertThat(invocations).isEqualTo(3) + } + @Test fun `other errors are not retried`() { // given @@ -104,6 +133,7 @@ internal class AnidbApiDownloaderTest { private val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { override fun string(key: String): String = "test" + override fun int(key: String): Int? = null } private fun respondingWith(body: () -> String) = object: HttpClient { diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index 3b5fcde6..9018fa5d 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -1,5 +1,8 @@ package io.github.manamiproject.modb.core.httpclient +import io.github.manamiproject.modb.core.config.ConfigRegistry +import io.github.manamiproject.modb.core.config.DefaultConfigRegistry +import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK import io.github.manamiproject.modb.core.coverage.KoverIgnore import io.github.manamiproject.modb.core.extensions.EMPTY @@ -53,6 +56,7 @@ import java.util.concurrent.TimeUnit.* * @property okhttpClient Instance of the OKHTTP client on which this client is based. * @property isTestContext Whether this runs in the unit test context or not. * @property headerCreator Creates default headers based on the selected browser type. + * @param configRegistry Source of `modb.core.httpclient.maxRetries`, the number of retries before a request fails. **Default:** `5` */ public class DefaultHttpClient( proxy: Proxy = NO_PROXY, @@ -71,9 +75,17 @@ public class DefaultHttpClient( private val isTestContext: Boolean = false, private val headerCreator: HeaderCreator = DefaultHeaderCreator.instance, useDefaultRetryCases: Boolean = true, + configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ) : HttpClient { - private val retryBehavior: RetryBehavior = RetryBehavior() + private val maxRetries: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = CONFIG_NAMESPACE, + default = DEFAULT_MAX_RETRIES, + validator = { it >= 0 }, + ) + + private val retryBehavior: RetryBehavior = RetryBehavior(maxAttempts = maxRetries) init { if (useDefaultRetryCases) retryBehavior.addCases( @@ -303,6 +315,18 @@ public class DefaultHttpClient( */ public const val DEFAULT_READ_TIMEOUT_IN_SECONDS: Long = 60L + /** + * Number of retries before a request fails, unless `modb.core.httpclient.maxRetries` sets another one. + * @since 15.0.0 + */ + public const val DEFAULT_MAX_RETRIES: Int = 5 + + /** + * Prefix of the properties read by this class. + * @since 15.0.0 + */ + public const val CONFIG_NAMESPACE: String = "modb.core.httpclient" + /** * Singleton of [DefaultHttpClient] * @since 15.0.0 diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt index 3bf0867a..106f7a49 100644 --- a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClientKtTest.kt @@ -4,6 +4,8 @@ import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.matching.EqualToJsonPattern import com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED +import io.github.manamiproject.modb.core.TestConfigRegistry +import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.extensions.EMPTY import io.github.manamiproject.modb.core.httpclient.HttpProtocol.HTTP_1_1 import io.github.manamiproject.modb.test.* @@ -372,6 +374,42 @@ internal class DefaultHttpClientKtTest : MockServerTestCase by W } } + @Test + fun `status code - number of retries can be set via maxRetries`() { + runTest { + // given + val path = "anime/1535" + + serverInstance.stubFor( + get(urlPathEqualTo("/$path")).willReturn( + aResponse() + .withHeader("Content-Type", "text/plain") + .withStatus(429) + .withBody(EMPTY) + ) + ) + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.core.httpclient.maxRetries") 2 else null + } + + val url = URI("http://localhost:$port/$path").toURL() + val client = DefaultHttpClient( + isTestContext = true, + configRegistry = testConfigRegistry, + ) + + // when + val result = exceptionExpected { + client.get(url) + } + + // then + assertThat(result).hasMessage("Execution failed despite [2] retry attempts. Last invocation of [GET http://localhost:$port/anime/1535] returned http status code [429]") + serverInstance.verify(3, getRequestedFor(urlPathEqualTo("/$path"))) + } + } + @Test fun `throwable - exception without a retry case is thrown as-is`() { runTest { From bddf01472e7cae9f38231b34a39f90753e6f855f Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 15:35:10 +0100 Subject: [PATCH 18/50] AniDB: dead entries from the title dump, downloaded at most once per day --- .../anidb/AnidbTitlesDumpHighestIdDetector.kt | 116 +++++++- .../modb/app/dataset/DeadEntriesAccessor.kt | 14 + .../app/dataset/DefaultDeadEntriesAccessor.kt | 36 +++ .../manamiproject/modb/app/TestingAssets.kt | 1 + .../AnidbTitlesDumpHighestIdDetectorTest.kt | 252 +++++++++++++++++- .../dataset/DefaultDeadEntriesAccessorTest.kt | 102 +++++++ 6 files changed, 497 insertions(+), 24 deletions(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt index c23f1036..998d2a4d 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt @@ -1,42 +1,74 @@ package io.github.manamiproject.modb.app.crawlers.anidb +import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.crawlers.HighestIdDetector +import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor +import io.github.manamiproject.modb.app.dataset.DefaultDeadEntriesAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DefaultDownloadControlStateAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK +import io.github.manamiproject.modb.core.extensions.RegularFile +import io.github.manamiproject.modb.core.extensions.regularFileExists import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.io.ByteArrayInputStream import java.net.URI +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.time.Duration +import java.time.Instant import java.util.zip.GZIPInputStream +import kotlin.coroutines.cancellation.CancellationException +import kotlin.io.path.getLastModifiedTime +import kotlin.io.path.readBytes +import kotlin.io.path.writeBytes /** - * Determines the highest anime id from the title dump which AniDB publishes. + * Determines the highest anime id from the title dump which AniDB publishes and derives the dead entries from it. * * The dump is a plain file rather than part of the website, so it is reachable without the * protection that anidb.net sits behind, and AniDB publishes it precisely so that clients do not * have to search or crawl for ids. It is regenerated daily and names every anime that exists, which - * also means ids that were never assigned are not requested at all. + * also means ids that were never assigned or have been deleted are not requested at all: every id + * from 1 to the highest id which the dump doesn't name becomes a dead entry. + * + * AniDB asks not to download the dump more than once per day. The last download is kept in the + * download control state directory and is only replaced once it is older than [maxAge]. * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. * @property httpClient To download the dump. * @property url Location of the dump. + * @property deadEntriesAccessor Access to dead entries files. + * @property downloadControlStateAccessor Access to DCS files. + * @property maxAge Age after which the kept dump is downloaded again. */ class AnidbTitlesDumpHighestIdDetector( + private val appConfig: Config = AppConfig.instance, private val httpClient: HttpClient = DefaultHttpClient(), private val url: URI = URI(TITLES_DUMP_URL), + private val deadEntriesAccessor: DeadEntriesAccessor = DefaultDeadEntriesAccessor.instance, + private val downloadControlStateAccessor: DownloadControlStateAccessor = DefaultDownloadControlStateAccessor.instance, + private val maxAge: Duration = Duration.ofDays(1), ): HighestIdDetector { - override suspend fun detectHighestId(): Int = withContext(LIMITED_NETWORK) { - log.info { "Fetching the AniDB title dump to determine the highest id." } + private val dumpAccess = Mutex() - val body = httpClient.get(url.toURL()).bodyAsByteArray() - val ids = parse(body) + override suspend fun detectHighestId(): Int = withContext(LIMITED_NETWORK) { + val ids = allIds() + val highestId = ids.max() - check(ids.isNotEmpty()) { "Title dump did not contain any anime id." } + log.info { "Title dump names [${ids.size}] anime, the highest id being [$highestId]." } - log.info { "Title dump names [${ids.size}] anime, the highest id being [${ids.max()}]." } + updateDeadEntries(ids, highestId) - return@withContext ids.max() + return@withContext highestId } /** @@ -45,9 +77,67 @@ class AnidbTitlesDumpHighestIdDetector( * @return Ids of all anime which exist on AniDB. */ suspend fun allIds(): Set = withContext(LIMITED_NETWORK) { - return@withContext parse(httpClient.get(url.toURL()).bodyAsByteArray()) + val ids = parse(dump()) + check(ids.isNotEmpty()) { "Title dump did not contain any anime id." } + return@withContext ids + } + + private suspend fun dump(): ByteArray = dumpAccess.withLock { + val file = dumpFile() + val now = appConfig.clock().instant() + val lastDownload = if (file.regularFileExists()) file.getLastModifiedTime().toInstant() else Instant.EPOCH + + if (Duration.between(lastDownload, now) < maxAge) { + log.info { "Using the AniDB title dump downloaded at [$lastDownload]. The next download is due after [${lastDownload.plus(maxAge)}]." } + return@withLock file.readBytes() + } + + log.info { "Downloading the AniDB title dump from [$url]." } + + try { + val body = httpClient.get(url.toURL()).bodyAsByteArray() + + // A broken or empty download must not replace a working dump, because the dead + // entries are derived from it. + check(parse(body).isNotEmpty()) { "Title dump did not contain any anime id." } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeBytes(body) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + + return@withLock body + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + if (!file.regularFileExists()) { + throw e + } + + log.warn { "Unable to download the AniDB title dump [${e.message}]. Using the one downloaded at [$lastDownload] instead." } + return@withLock file.readBytes() + } + } + + private suspend fun updateDeadEntries(ids: Set, highestId: Int) { + // Each dead entry removes the corresponding DCS file. If the dump names fewer anime than + // the dataset already has, it is more likely to be broken than AniDB having deleted them. + val highestIdAlreadyInDataset = downloadControlStateAccessor.highestIdAlreadyInDataset(AnidbConfig) + + if (highestId < highestIdAlreadyInDataset) { + log.warn { "Not updating the [${AnidbConfig.hostname()}] dead entries, because the highest id of the title dump [$highestId] is smaller than the highest id already in the dataset [$highestIdAlreadyInDataset]." } + return + } + + val deadEntries = (1..highestId) + .filterNot { ids.contains(it) } + .map { it.toString() } + .toSet() + + deadEntriesAccessor.replaceDeadEntries(deadEntries, AnidbConfig) } + private fun dumpFile(): RegularFile = appConfig.downloadControlStateDirectory().resolve(DUMP_FILE_NAME) + private fun parse(body: ByteArray): Set { // The file is gzipped content rather than a compressed transfer, so the http client has no // reason to unwrap it. @@ -77,6 +167,12 @@ class AnidbTitlesDumpHighestIdDetector( */ const val TITLES_DUMP_URL = "https://anidb.net/api/anime-titles.dat.gz" + /** + * Name of the file in the download control state directory which keeps the last download of the dump. + * @since 1.0.0 + */ + const val DUMP_FILE_NAME = "anidb-anime-titles.dat.gz" + /** * Singleton of [AnidbTitlesDumpHighestIdDetector]. * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt index 1e2488f3..a9a58437 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DeadEntriesAccessor.kt @@ -38,6 +38,20 @@ interface DeadEntriesAccessor { */ suspend fun removeDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) + /** + * Replaces the whole dead entries list of a metadata provider. + * For metadata providers which publish a complete list of their anime. Ids which are new to the list are + * handled like [addDeadEntry], ids which are no longer part of it like [removeDeadEntry]. + * @since 1.0.0 + * @param animeIds Complete list of dead entries. + * @param metaDataProviderConfig Configuration for a specific metadata provider. + */ + suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + val current = fetchDeadEntries(metaDataProviderConfig) + (current - animeIds).forEach { removeDeadEntry(it, metaDataProviderConfig) } + (animeIds - current).forEach { addDeadEntry(it, metaDataProviderConfig) } + } + /** * Find dead entries in a list of [URI]. * @since 1.0.0 diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt index dab01280..c819fbca 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessor.kt @@ -121,6 +121,42 @@ class DefaultDeadEntriesAccessor( } } + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + if (!appConfig.deadEntriesSupported(metaDataProviderConfig)) { + return + } + + if (!isInitialized) { + init() + } + + val added = writeAccess.withLock { + val deadEntriesOfProvider = when (metaDataProviderConfig.hostname()) { + AnidbConfig.hostname() -> deadEntries.anidb + AnilistConfig.hostname() -> deadEntries.anilist + AnimenewsnetworkConfig.hostname() -> deadEntries.animenewsnetwork + KitsuConfig.hostname() -> deadEntries.kitsu + MyanimelistConfig.hostname() -> deadEntries.myanimelist + else -> throw IllegalStateException("Metadata provider [${metaDataProviderConfig.hostname()}] is not supported.") + } + + val added = animeIds - deadEntriesOfProvider + val removed = deadEntriesOfProvider - animeIds + + log.info { "[${metaDataProviderConfig.hostname()}] has [${animeIds.size}] dead entries. Adding [${added.size}] and removing [${removed.size}]." } + + if (added.isNotEmpty() || removed.isNotEmpty()) { + deadEntriesOfProvider.clear() + deadEntriesOfProvider.addAll(animeIds) + jsonSerializer.serialize(deadEntriesOfProvider, minify = true).writeToFile(deadEntriesFile(metaDataProviderConfig, JSON_MINIFIED)) + } + + added + } + + added.forEach { downloadControlStateAccessor.removeDeadEntry(metaDataProviderConfig, it) } + } + override suspend fun determineDeadEntries(sources: Collection): Set { if (!isInitialized) { init() diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt index 312573ff..07665e2c 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt @@ -159,6 +159,7 @@ internal object TestDeadEntriesAccessor: DeadEntriesAccessor { override fun deadEntriesFile(metaDataProviderConfig: MetaDataProviderConfig, type: DatasetFileType): RegularFile = shouldNotBeInvoked() override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() override suspend fun removeDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { } + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() override suspend fun determineDeadEntries(sources: Collection): Set = shouldNotBeInvoked() override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = shouldNotBeInvoked() } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt index ae739b45..7c025063 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt @@ -1,28 +1,58 @@ package io.github.manamiproject.modb.app.crawlers.anidb +import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.TestDeadEntriesAccessor +import io.github.manamiproject.modb.app.TestDownloadControlStateAccessor +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.AnidbTitlesDumpHighestIdDetector.Companion.DUMP_FILE_NAME +import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor +import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor +import io.github.manamiproject.modb.core.config.AnimeId +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig +import io.github.manamiproject.modb.core.extensions.Directory import io.github.manamiproject.modb.core.httpclient.HttpClient import io.github.manamiproject.modb.core.httpclient.HttpResponse import io.github.manamiproject.modb.core.httpclient.RequestBody import io.github.manamiproject.modb.core.httpclient.RetryCase import io.github.manamiproject.modb.test.exceptionExpected import io.github.manamiproject.modb.test.shouldNotBeInvoked +import io.github.manamiproject.modb.test.tempDirectory import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested +import java.io.IOException import java.net.URL +import java.nio.file.attribute.FileTime +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.io.path.readBytes +import kotlin.io.path.setLastModifiedTime +import kotlin.io.path.writeBytes import kotlin.test.Test internal class AnidbTitlesDumpHighestIdDetectorTest { + private val now = Instant.parse("2026-10-29T12:00:00Z") + private val smallDump = """ + # created: whenever + 1|1|x-jat|First + 3|1|x-jat|Third + 3|4|en|Third in english + 5|1|x-jat|Fifth + """.trimIndent().toByteArray() + @Nested inner class DetectHighestIdTests { @Test fun `reads the highest id from the gzipped dump`() { - runBlocking { + tempDirectory { // given val dump = this::class.java.classLoader.getResourceAsStream("crawlers/anidb/anime-titles.dat.gz")!!.readBytes() - val detector = AnidbTitlesDumpHighestIdDetector(httpClient = clientReturning(dump)) + val detector = detector(tempDir, clientReturning(dump)) // when val result = detector.detectHighestId() @@ -34,10 +64,10 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { @Test fun `ignores comments and reads every id`() { - runBlocking { + tempDirectory { // given val dump = this::class.java.classLoader.getResourceAsStream("crawlers/anidb/anime-titles.dat.gz")!!.readBytes() - val detector = AnidbTitlesDumpHighestIdDetector(httpClient = clientReturning(dump)) + val detector = detector(tempDir, clientReturning(dump)) // when val result = detector.allIds() @@ -51,19 +81,201 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { @Test fun `throws if the dump contains no id`() { - // given - val detector = AnidbTitlesDumpHighestIdDetector( - httpClient = clientReturning("# created: whenever\n".toByteArray()), - ) - - // when - val result = exceptionExpected { - detector.detectHighestId() + tempDirectory { + // given + val detector = detector(tempDir, clientReturning("# created: whenever\n".toByteArray())) + + // when + val result = exceptionExpected { + detector.detectHighestId() + } + + // then + assertThat(result).hasMessage("Title dump did not contain any anime id.") + } + } + + @Test + fun `every id up to the highest id which the dump doesn't name becomes a dead entry`() { + tempDirectory { + // given + var replacedWith = emptySet() + var replacedFor: MetaDataProviderConfig? = null + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + replacedWith = animeIds + replacedFor = metaDataProviderConfig + } + } + + val detector = detector(tempDir, clientReturning(smallDump), deadEntriesAccessor = testDeadEntriesAccessor) + + // when + val result = detector.detectHighestId() + + // then + assertThat(result).isEqualTo(5) + assertThat(replacedWith).containsExactlyInAnyOrder("2", "4") + assertThat(replacedFor).isEqualTo(AnidbConfig) + } + } + + @Test + fun `doesn't touch the dead entries if the dump ends below the highest id already in the dataset`() { + tempDirectory { + // given + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) = shouldNotBeInvoked() + } + + val detector = detector( + tempDir = tempDir, + httpClient = clientReturning(smallDump), + deadEntriesAccessor = testDeadEntriesAccessor, + highestIdAlreadyInDataset = 6, + ) + + // when + val result = detector.detectHighestId() + + // then + assertThat(result).isEqualTo(5) + } + } + } + + @Nested + inner class DownloadOncePerDayTests { + + @Test + fun `keeps the downloaded dump in the download control state directory`() { + tempDirectory { + // given + val detector = detector(tempDir, clientReturning(smallDump)) + + // when + detector.allIds() + + // then + assertThat(tempDir.resolve(DUMP_FILE_NAME).readBytes()).isEqualTo(smallDump) + } + } + + @Test + fun `uses the kept dump without downloading if it is younger than one day`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofHours(23)))) + + val detector = detector(tempDir, notInvokedClient()) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + } + } + + @Test + fun `downloads again if the kept dump is one day old`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(1)))) + + val newDump = "1|1|x-jat|First\n7|1|x-jat|Seventh\n".toByteArray() + val detector = detector(tempDir, clientReturning(newDump)) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 7) + assertThat(file.readBytes()).isEqualTo(newDump) } + } + + @Test + fun `uses the kept dump if the download fails`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(3)))) + + val detector = detector(tempDir, failingClient()) - // then - assertThat(result).hasMessage("Title dump did not contain any anime id.") + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + } } + + @Test + fun `doesn't replace the kept dump with a download without ids`() { + tempDirectory { + // given + val file = tempDir.resolve(DUMP_FILE_NAME) + file.writeBytes(smallDump) + file.setLastModifiedTime(FileTime.from(now.minus(Duration.ofDays(3)))) + + val detector = detector(tempDir, clientReturning("maintenance".toByteArray())) + + // when + val result = detector.allIds() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 3, 5) + assertThat(file.readBytes()).isEqualTo(smallDump) + } + } + + @Test + fun `throws if the download fails and there is no kept dump`() { + tempDirectory { + // given + val detector = detector(tempDir, failingClient()) + + // when + val result = exceptionExpected { + detector.allIds() + } + + // then + assertThat(result).hasMessage("unreachable") + } + } + } + + private fun detector( + tempDir: Directory, + httpClient: HttpClient, + deadEntriesAccessor: DeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { } + }, + highestIdAlreadyInDataset: Int = 0, + ): AnidbTitlesDumpHighestIdDetector { + val testAppConfig = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = tempDir + override fun clock(): Clock = Clock.fixed(now, UTC) + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = highestIdAlreadyInDataset + } + + return AnidbTitlesDumpHighestIdDetector( + appConfig = testAppConfig, + httpClient = httpClient, + deadEntriesAccessor = deadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + ) } private fun clientReturning(body: ByteArray): HttpClient = object: HttpClient { @@ -73,4 +285,16 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this } + + private fun notInvokedClient(): HttpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + } + + private fun failingClient(): HttpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = throw IOException("unreachable") + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + } } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt index 1be69cfe..44bd82c2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/dataset/DefaultDeadEntriesAccessorTest.kt @@ -846,6 +846,108 @@ internal class DefaultDeadEntriesAccessorTest { } } + @Nested + inner class ReplaceDeadEntriesTests { + + @Test + fun `replaces the list, writes the file and removes the dcs entries of new dead entries only`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnidbConfig) + } + + val removedFromDcs = mutableListOf() + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun removeDeadEntry(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId) { + removedFromDcs.add(animeId) + } + } + + val testDeserializer = object: Deserializer by TestDeserializer() { + override suspend fun deserialize(source: RegularFile): DeadEntries = DeadEntries( + `$schema` = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/refs/heads/master/dead-entries/dead-entries.schema.json"), + lastUpdate = "2024-08-01", + deadEntries = listOf("111", "222"), + ) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + jsonDeserializer = testDeserializer, + downloadControlStateAccessor = testDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("222", "333", "444"), AnidbConfig) + + // then + assertThat(deadEntriesAccessor.fetchDeadEntries(AnidbConfig)).containsExactlyInAnyOrder("222", "333", "444") + assertThat(removedFromDcs).containsExactlyInAnyOrder("333", "444") + val result = FromRegularFileDeserializer( + deserializer = DeadEntriesFromInputStreamDeserializer.instance).deserialize( + deadEntriesAccessor.deadEntriesFile(AnidbConfig, JSON_MINIFIED), + ).deadEntries + assertThat(result).containsExactlyInAnyOrder("222", "333", "444") + } + } + + @Test + fun `doesn't write the file if nothing changed`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnidbConfig) + } + + val testDeserializer = object: Deserializer by TestDeserializer() { + override suspend fun deserialize(source: RegularFile): DeadEntries = DeadEntries( + `$schema` = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/refs/heads/master/dead-entries/dead-entries.schema.json"), + lastUpdate = "2024-08-01", + deadEntries = listOf("111"), + ) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + jsonDeserializer = testDeserializer, + downloadControlStateAccessor = TestDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("111"), AnidbConfig) + + // then + assertThat(deadEntriesAccessor.deadEntriesFile(AnidbConfig, JSON_MINIFIED)).doesNotExist() + assertThat(deadEntriesAccessor.fetchDeadEntries(AnidbConfig)).containsExactly("111") + } + } + + @Test + fun `does nothing for a metadata provider without dead entry files`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun outputDirectory(): Directory = tempDir + override fun metaDataProviderConfigurations(): Set = setOf(AnisearchConfig) + } + + val deadEntriesAccessor = DefaultDeadEntriesAccessor( + appConfig = testAppConfig, + downloadControlStateAccessor = TestDownloadControlStateAccessor, + ) + + // when + deadEntriesAccessor.replaceDeadEntries(setOf("111"), AnisearchConfig) + + // then + assertThat(tempDir.resolve("dead-entries")).doesNotExist() + } + } + } + @Nested inner class DetermineDeadEntriesTests { From 12308b6b3613c37a351fe7a37e89b26b9e3afb1b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 16:00:26 +0100 Subject: [PATCH 19/50] Offer to create config.toml from the template if none is found --- app/build.gradle.kts | 5 + .../io/github/manamiproject/modb/app/App.kt | 11 +- .../manamiproject/modb/app/ConfigFileSetup.kt | 100 ++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8aa7f756..c75d426f 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -63,6 +63,11 @@ application { } tasks { + // Bundled so that the app can offer to create a config.toml from it on the first start. + named("processResources") { + from(rootProject.file("config.example.toml")) + } + named("shadowJar") { archiveBaseName.set("modb-app") archiveClassifier.set("") diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index ee4215f9..74555375 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -35,10 +35,17 @@ import javax.swing.JPasswordField import javax.swing.SwingUtilities import kotlin.system.exitProcess +// Lazy, because the first logger sets up logback, which reads the configuration. That has to wait +// until ensureConfigFileExists() had the chance to create it. @KoverIgnore -private val log = org.slf4j.LoggerFactory.getLogger("App") +private val log by lazy { org.slf4j.LoggerFactory.getLogger("App") } -fun main() = runCoroutine { +fun main() { + ensureConfigFileExists() + crawlAndCreateDataset() +} + +private fun crawlAndCreateDataset() = runCoroutine { val disabled = DefaultConfigRegistry.instance.list("modb.app.disabledCrawlers")?.toSet() ?: emptySet() // Only the crawlers of these sites change the outgoing address by restarting the network device. diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt new file mode 100644 index 00000000..c2824afb --- /dev/null +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/ConfigFileSetup.kt @@ -0,0 +1,100 @@ +package io.github.manamiproject.modb.app + +import io.github.manamiproject.modb.core.coverage.KoverIgnore +import io.github.manamiproject.modb.serde.json.serializer.DeadEntriesJsonSerializer +import kotlinx.coroutines.runBlocking +import java.nio.file.Path +import kotlin.io.path.Path +import kotlin.io.path.absolute +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.writeText +import kotlin.system.exitProcess + +private const val CONFIG_FILE = "config.toml" +private const val CONFIG_LOCATION_KEY = "modb.core.config.location" +private const val TEMPLATE_RESOURCE = "config.example.toml" +private val DEAD_ENTRIES_FILES = listOf("anidb", "anilist", "animenewsnetwork", "kitsu", "myanimelist") + +/** + * Offers to create a `config.toml` from the template if none can be found. + * + * Has to run before anything logs: logback determines its log directory from the configuration, and + * the configuration is read only once, so a file created afterwards would not be seen. + */ +@KoverIgnore +internal fun ensureConfigFileExists() { + if (configFileExists()) { + return + } + + val workingDirectory = Path("").absolute() + val configFile = workingDirectory.resolve(CONFIG_FILE) + + println("No [$CONFIG_FILE] found in [$workingDirectory].") + println("Either start the app from the folder containing your [$CONFIG_FILE] or pass its location with -D$CONFIG_LOCATION_KEY=/path/to/$CONFIG_FILE.") + println() + println("Create [$configFile] from the default template instead?") + println(" downloadsDirectory = [${workingDirectory.resolve("downloads")}]") + println(" outputDirectory = [${workingDirectory.resolve("output")}]") + println(" downloadControlStateDirectory = [${workingDirectory.resolve("dcs")}]") + println(" disabledCrawlers = [anidb.net]") + print("Create it? [y/N]: ") + System.out.flush() + + val answer = readlnOrNull()?.trim()?.lowercase() + + if (answer != "y" && answer != "yes") { + println("No config created. Exiting.") + exitProcess(1) + } + + createDefaultConfig(workingDirectory, configFile) + println("Created [$configFile]. Add your AniDB client to it before enabling the anidb.net crawler.") + println() +} + +private fun configFileExists(): Boolean { + val overriddenLocation = System.getProperty(CONFIG_LOCATION_KEY) ?: System.getenv(CONFIG_LOCATION_KEY) + + // An explicitly given location is the caller's choice. If it is wrong, the config registry reports it. + if (!overriddenLocation.isNullOrBlank()) { + return true + } + + return Path(CONFIG_FILE).exists() || Thread.currentThread().contextClassLoader.getResource(CONFIG_FILE) != null +} + +private fun createDefaultConfig(workingDirectory: Path, configFile: Path) { + val downloads = workingDirectory.resolve("downloads").createDirectories() + val output = workingDirectory.resolve("output").createDirectories() + val dcs = workingDirectory.resolve("dcs").createDirectories() + + val template = Thread.currentThread().contextClassLoader.getResourceAsStream(TEMPLATE_RESOURCE) + ?.bufferedReader() + ?.use { it.readText() } + ?: error("Template [$TEMPLATE_RESOURCE] is missing from the app.") + + val config = template + .setValue("downloadsDirectory", downloads) + .setValue("outputDirectory", output) + .setValue("downloadControlStateDirectory", dcs) + + configFile.writeText(config) + + // The dataset expects a dead entries file for each provider which has one, even on the very first run. + val deadEntriesDirectory = output.resolve("dead-entries").createDirectories() + val emptyDeadEntries = runBlocking { DeadEntriesJsonSerializer.instance.serialize(emptyList(), minify = true) } + + DEAD_ENTRIES_FILES + .map { deadEntriesDirectory.resolve("$it-minified.json") } + .filterNot { it.exists() } + .forEach { it.writeText(emptyDeadEntries) } +} + +private fun String.setValue(key: String, directory: Path): String { + val value = directory.toString().replace("\\", "\\\\").replace("\"", "\\\"") + val line = Regex("""(?m)^(\s*$key\s*=\s*)"[^"]*"""") + check(line.containsMatchIn(this)) { "Template doesn't contain [$key]." } + return line.replace(this) { "${it.groupValues[1]}\"$value\"" } +} From 640173aff0f015185ed1f2d12cc0b2e40d6cd9b7 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 16:21:40 +0100 Subject: [PATCH 20/50] Clean up empty conv files at startup, skip broken ones and log AniDB bans without a stack trace --- .../io/github/manamiproject/modb/app/App.kt | 4 ++ .../DependentConversionWatchService.kt | 9 ++++ .../modb/app/convfiles/FileConverter.kt | 17 +++++++ .../convfiles/SimpleConversionWatchService.kt | 9 ++++ .../DefaultDownloadControlStateUpdater.kt | 20 +++++++- .../SimpleConversionWatchServiceTest.kt | 47 +++++++++++++++++++ .../DefaultDownloadControlStateUpdaterTest.kt | 47 +++++++++++++++++++ 7 files changed, 151 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 74555375..95dd34c6 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app +import io.github.manamiproject.modb.anidb.AnidbBannedException import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig @@ -209,6 +210,9 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: jobs.add(launch { try { block.invoke() + } catch (_: AnidbBannedException) { + // Expected while a ban lasts, so a stack trace adds nothing. + log.error("Crawler for [$hostname] stopped, because AniDB still answers with [banned] after all retries. Everything downloaded so far is kept. Wait until the ban has been lifted or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { log.error("Crawler for [$hostname] stopped: ${e.message}", e) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt index fa20a58f..875f9518 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/DependentConversionWatchService.kt @@ -69,6 +69,7 @@ class DependentConversionWatchService( validFileSuffixes.addAll(metaDataConfigs.map { it.value.fileSuffix() }) } workingDirs.values.forEach { deleteLocks(it) } + workingDirs.values.forEach { cleanUpEmptyConvertedFiles(it) } dependentFileConverter.convertUnconvertedFiles() isPrepared = true } @@ -91,6 +92,14 @@ class DependentConversionWatchService( jobs.forEach { it.cancelAndJoin() } } + private suspend fun cleanUpEmptyConvertedFiles(workingDir: Directory) { + val deleted = workingDir.deleteEmptyConvertedFiles() + + if (deleted > 0) { + log.warn { "Deleted [$deleted] empty [*.$CONVERTED_FILE_SUFFIX] files in [$workingDir]. They are converted again." } + } + } + private suspend fun deleteLocks(workingDir: Directory) { log.info { "Deleting [$LOCK_FILE_SUFFIX] files in [$workingDir]" } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt index 2de320b8..9616045e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/FileConverter.kt @@ -1,7 +1,11 @@ package io.github.manamiproject.modb.app.convfiles +import io.github.manamiproject.modb.core.extensions.Directory import io.github.manamiproject.modb.core.extensions.RegularFile +import io.github.manamiproject.modb.core.extensions.listRegularFiles import io.github.manamiproject.modb.core.anime.AnimeRaw +import kotlin.io.path.deleteIfExists +import kotlin.io.path.fileSize /** * File suffix used for the files containing the serialized [AnimeRaw] object as intermediate format. @@ -9,6 +13,19 @@ import io.github.manamiproject.modb.core.anime.AnimeRaw */ const val CONVERTED_FILE_SUFFIX = "conv" +/** + * Deletes empty converted files. Stopping the app while a file is being converted can leave an empty file behind, + * which would otherwise count as converted. Without it, the raw file is converted again. + * @since 1.0.0 + * @receiver Working directory of a metadata provider. + * @return Number of deleted files. + */ +internal suspend fun Directory.deleteEmptyConvertedFiles(): Int { + return listRegularFiles("*.$CONVERTED_FILE_SUFFIX") + .filter { it.fileSize() == 0L } + .count { it.deleteIfExists() } +} + /** * Converts raw files into a generic intermediate format based on [AnimeRaw]. diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt index 7716a289..c663810c 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchService.kt @@ -45,6 +45,7 @@ class SimpleConversionWatchService @KoverIgnore constructor( override suspend fun prepare() = withContext(LIMITED_FS) { deleteLocks() + cleanUpEmptyConvertedFiles() fileConverter.convertUnconvertedFiles() workingDir.register(watchService, ENTRY_CREATE) isPrepared = true @@ -82,6 +83,14 @@ class SimpleConversionWatchService @KoverIgnore constructor( watchService.close() } + private suspend fun cleanUpEmptyConvertedFiles() { + val deleted = workingDir.deleteEmptyConvertedFiles() + + if (deleted > 0) { + log.warn { "Deleted [$deleted] empty [*.$CONVERTED_FILE_SUFFIX] files in [$workingDir]. They are converted again." } + } + } + private suspend fun deleteLocks() { log.info { "Deleting [$LOCK_FILE_SUFFIX] files in [$workingDir]" } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt index a48aba5a..10139225 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdater.kt @@ -134,10 +134,26 @@ class DefaultDownloadControlStateUpdater( .map { workDir -> workDir.listRegularFiles("*.$CONVERTED_FILE_SUFFIX").map { file -> async { - Json.parseJson(file.readFile())!! to file.fileName() + val anime = try { + Json.parseJson(file.readFile()) + } catch (e: Exception) { + log.debug { "Unable to parse [$file]: ${e.message}" } + null + } + + when (anime) { + null -> { + // Most likely written only partially, because the app was stopped during the + // conversion. Without the file the raw file counts as unconverted again. + file.deleteIfExists() + log.warn { "Skipping [${file.fileName()}] of [${workDir.fileName()}], because it is empty or broken. It has been deleted and will be converted again on the next start." } + null + } + else -> anime to file.fileName() + } } } - }.flatten().awaitAll() + }.flatten().awaitAll().filterNotNull() return@withContext jobsResults } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt index cddaf5ff..68019a90 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/convfiles/SimpleConversionWatchServiceTest.kt @@ -24,6 +24,9 @@ import org.junit.jupiter.api.Nested import java.nio.file.Path import kotlin.io.path.createDirectory import kotlin.io.path.createFile +import kotlin.io.path.fileSize +import kotlin.io.path.readText +import kotlin.io.path.writeText import kotlin.io.path.deleteIfExists import kotlin.io.path.listDirectoryEntries import kotlin.test.Test @@ -78,6 +81,50 @@ internal class SimpleConversionWatchServiceTest { } } + @Test + fun `deletes empty conv files when starting the watch service and converts their raw files again`() { + tempDirectory { + // given + val anime = AnimeRaw("Death Note") + + val testAppConfig = object: Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + override fun fileSuffix(): FileSuffix = "html" + } + + val convertedPaths = mutableListOf() + val testConverter = object: PathAnimeConverter by TestPathAnimeConverter { + override suspend fun convert(path: Path): List { + convertedPaths.add(path) + return listOf(anime) + } + } + + tempDir.resolve("1535.${testConfig.fileSuffix()}").createFile() + val emptyConvFile = tempDir.resolve("1535.$CONVERTED_FILE_SUFFIX").createFile() + tempDir.resolve("4000.${testConfig.fileSuffix()}").createFile() + tempDir.resolve("4000.$CONVERTED_FILE_SUFFIX").writeText("{}") + + val simpleConversionWatchService = SimpleConversionWatchService( + appConfig = testAppConfig, + converter = testConverter, + metaDataProviderConfig = testConfig, + ) + + // when + simpleConversionWatchService.prepare() + + // then + assertThat(convertedPaths.map { it.fileName() }).containsExactly("1535.${testConfig.fileSuffix()}") + assertThat(emptyConvFile.fileSize()).isGreaterThan(0L) + assertThat(tempDir.resolve("4000.$CONVERTED_FILE_SUFFIX").readText()).isEqualTo("{}") + } + } + @Test fun `convert all unconverted files at startup`() { tempDirectory { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt index 51db9fe6..71d780eb 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt @@ -472,6 +472,53 @@ internal class DefaultDownloadControlStateUpdaterTest { } } + @Test + fun `skips and deletes an empty conv file instead of failing the run`() { + tempDirectory { + // given + val rawdata = tempDir.resolve("rawdata").createDirectory() + val dcs = tempDir.resolve("dcs").createDirectory() + + val anime = AnimeRaw( + _title = "Shin Seiki Evangelion Movie: THE END OF EVANGELION", + _sources = hashSetOf(URI("https://anilist.co/anime/32")), + ) + Json.toJson(anime).writeToFile(rawdata.resolve("32.$CONVERTED_FILE_SUFFIX").createFile()) + val emptyFile = rawdata.resolve("64.$CONVERTED_FILE_SUFFIX").createFile() + + val testAppConfig = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dcs + override fun metaDataProviderConfigurations(): Set = setOf(AnilistConfig) + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = rawdata + override fun findMetaDataProviderConfig(host: Hostname): MetaDataProviderConfig = AnilistConfig + override fun clock(): Clock = Clock.fixed(Instant.parse("2021-11-02T17:55:43.035Z"), ZoneId.systemDefault()) + } + + val updatedIds = mutableListOf() + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun allDcsEntries(metaDataProviderConfig: MetaDataProviderConfig): List = emptyList() + override fun downloadControlStateDirectory(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + override suspend fun dcsEntryExists(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId): Boolean = false + override suspend fun createOrUpdate(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId, downloadControlStateEntry: DownloadControlStateEntry): Boolean { + updatedIds.add(animeId) + return true + } + } + + val downloadControlStateUpdater = DefaultDownloadControlStateUpdater( + appConfig = testAppConfig, + downloadControlStateAccessor = testDownloadControlStateAccessor, + ) + + // when + downloadControlStateUpdater.updateAll() + + // then + assertThat(updatedIds).containsExactly("32") + assertThat(emptyFile).doesNotExist() + } + } + @Test fun `creates dcs file if it doesn't exist`() { tempDirectory { From e2e141c0b4226dc3a1112be552df52d7a6275f12 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 16:36:22 +0100 Subject: [PATCH 21/50] Allow native access for zstd-jni to silence the JDK warning --- analyzer/build.gradle.kts | 2 ++ anidb/build.gradle.kts | 2 ++ anilist/build.gradle.kts | 2 ++ anime-planet/build.gradle.kts | 2 ++ animenewsnetwork/build.gradle.kts | 2 ++ anisearch/build.gradle.kts | 2 ++ app/build.gradle.kts | 4 ++++ core/build.gradle.kts | 2 ++ kitsu/build.gradle.kts | 2 ++ kommand/build.gradle.kts | 2 ++ lib/build.gradle.kts | 2 ++ livechart/build.gradle.kts | 2 ++ myanimelist/build.gradle.kts | 2 ++ serde/build.gradle.kts | 2 ++ simkl/build.gradle.kts | 2 ++ test/build.gradle.kts | 2 ++ 16 files changed, 34 insertions(+) diff --git a/analyzer/build.gradle.kts b/analyzer/build.gradle.kts index 02c83e57..acf6e2a4 100644 --- a/analyzer/build.gradle.kts +++ b/analyzer/build.gradle.kts @@ -51,6 +51,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/anidb/build.gradle.kts b/anidb/build.gradle.kts index 7e854401..29edc11a 100644 --- a/anidb/build.gradle.kts +++ b/anidb/build.gradle.kts @@ -52,6 +52,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/anilist/build.gradle.kts b/anilist/build.gradle.kts index b66152d1..b4cb9d96 100644 --- a/anilist/build.gradle.kts +++ b/anilist/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/anime-planet/build.gradle.kts b/anime-planet/build.gradle.kts index 825451a3..9e052e0d 100644 --- a/anime-planet/build.gradle.kts +++ b/anime-planet/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/animenewsnetwork/build.gradle.kts b/animenewsnetwork/build.gradle.kts index 55d80505..44f751a7 100644 --- a/animenewsnetwork/build.gradle.kts +++ b/animenewsnetwork/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/anisearch/build.gradle.kts b/anisearch/build.gradle.kts index fe6c54cc..1c79a220 100644 --- a/anisearch/build.gradle.kts +++ b/anisearch/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/app/build.gradle.kts b/app/build.gradle.kts index c75d426f..f85da332 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -50,6 +50,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int @@ -74,6 +76,8 @@ tasks { archiveVersion.set("") manifest { attributes["Main-Class"] = mainClassPath + // zstd-jni loads a native library, which JDK 24+ only allows silently with this attribute. + attributes["Enable-Native-Access"] = "ALL-UNNAMED" } exclude(".gitemptydir") } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 9d10e14f..a5580f32 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -61,6 +61,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/kitsu/build.gradle.kts b/kitsu/build.gradle.kts index 00e74866..0fad5e74 100644 --- a/kitsu/build.gradle.kts +++ b/kitsu/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/kommand/build.gradle.kts b/kommand/build.gradle.kts index b22a4988..44eec7c2 100644 --- a/kommand/build.gradle.kts +++ b/kommand/build.gradle.kts @@ -38,6 +38,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = Runtime.getRuntime().availableProcessors() diff --git a/lib/build.gradle.kts b/lib/build.gradle.kts index 173fcd59..277f7ba7 100644 --- a/lib/build.gradle.kts +++ b/lib/build.gradle.kts @@ -63,6 +63,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/livechart/build.gradle.kts b/livechart/build.gradle.kts index 59fde0b7..1a42b57f 100644 --- a/livechart/build.gradle.kts +++ b/livechart/build.gradle.kts @@ -44,6 +44,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/myanimelist/build.gradle.kts b/myanimelist/build.gradle.kts index 85685815..22d4b828 100644 --- a/myanimelist/build.gradle.kts +++ b/myanimelist/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/serde/build.gradle.kts b/serde/build.gradle.kts index 31c1adce..0dfafeca 100644 --- a/serde/build.gradle.kts +++ b/serde/build.gradle.kts @@ -44,6 +44,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/simkl/build.gradle.kts b/simkl/build.gradle.kts index f41adaa4..dd946838 100644 --- a/simkl/build.gradle.kts +++ b/simkl/build.gradle.kts @@ -42,6 +42,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int diff --git a/test/build.gradle.kts b/test/build.gradle.kts index c91e8f93..53c25b3c 100644 --- a/test/build.gradle.kts +++ b/test/build.gradle.kts @@ -46,6 +46,8 @@ tasks.withType().configureEach tasks.withType { useJUnitPlatform() + // zstd-jni loads a native library, which JDK 24+ only allows silently with this flag. + jvmArgs("--enable-native-access=ALL-UNNAMED") reports.html.required.set(false) reports.junitXml.required.set(true) maxParallelForks = rootProject.extra["maxParallelForks"] as Int From 7a6b19a6bca066849a9bd22ba2ed18ccd0ef5337 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 16:41:44 +0100 Subject: [PATCH 22/50] Readable timestamps in the AniDB title dump log --- .../anidb/AnidbTitlesDumpHighestIdDetector.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt index 998d2a4d..a0cafbd1 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt @@ -24,6 +24,7 @@ import java.nio.file.StandardCopyOption.ATOMIC_MOVE import java.nio.file.StandardCopyOption.REPLACE_EXISTING import java.time.Duration import java.time.Instant +import java.time.format.DateTimeFormatter import java.util.zip.GZIPInputStream import kotlin.coroutines.cancellation.CancellationException import kotlin.io.path.getLastModifiedTime @@ -88,7 +89,8 @@ class AnidbTitlesDumpHighestIdDetector( val lastDownload = if (file.regularFileExists()) file.getLastModifiedTime().toInstant() else Instant.EPOCH if (Duration.between(lastDownload, now) < maxAge) { - log.info { "Using the AniDB title dump downloaded at [$lastDownload]. The next download is due after [${lastDownload.plus(maxAge)}]." } + val nextDownload = lastDownload.plus(maxAge) + log.info { "Using the AniDB title dump from [${format(lastDownload)}]. Next download possible from [${format(nextDownload)}] (in [${formatDuration(Duration.between(now, nextDownload))}])." } return@withLock file.readBytes() } @@ -113,7 +115,7 @@ class AnidbTitlesDumpHighestIdDetector( throw e } - log.warn { "Unable to download the AniDB title dump [${e.message}]. Using the one downloaded at [$lastDownload] instead." } + log.warn { "Unable to download the AniDB title dump [${e.message}]. Using the one from [${format(lastDownload)}] instead." } return@withLock file.readBytes() } } @@ -136,6 +138,10 @@ class AnidbTitlesDumpHighestIdDetector( deadEntriesAccessor.replaceDeadEntries(deadEntries, AnidbConfig) } + private fun format(instant: Instant): String = TIMESTAMP_FORMAT.format(instant.atZone(appConfig.clock().zone)) + + private fun formatDuration(duration: Duration): String = "%dh %02dm".format(duration.toHours(), duration.toMinutesPart()) + private fun dumpFile(): RegularFile = appConfig.downloadControlStateDirectory().resolve(DUMP_FILE_NAME) private fun parse(body: ByteArray): Set { @@ -157,6 +163,7 @@ class AnidbTitlesDumpHighestIdDetector( private val log by LoggerDelegate() private const val COMMENT = "#" + private val TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm") private const val SEPARATOR = '|' private const val GZIP_MAGIC_FIRST: Byte = 0x1f private const val GZIP_MAGIC_SECOND: Byte = 0x8b.toByte() From ac214aa427f5d6e7c87662b7990ff696e4091afd Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 18:29:45 +0100 Subject: [PATCH 23/50] Skip the incorrect sudo password test if sudo has cached credentials --- .../github/manamiproject/kommand/JavaProcessBuilderTest.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt index 29ba8562..db2d1a55 100644 --- a/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt +++ b/kommand/src/test/kotlin/io/github/manamiproject/kommand/JavaProcessBuilderTest.kt @@ -226,8 +226,9 @@ internal class JavaProcessBuilderTest { private fun sudoRequiresPassword(): Boolean { return try { - // -k ignores cached credentials, -n fails instead of asking if a password is required - ProcessBuilder("sudo", "-k", "-n", "true") + // -n fails instead of asking if a password is required. Without -k, so that credentials cached by an earlier + // sudo in the same terminal count as well: the command under test would use them and never check the password. + ProcessBuilder("sudo", "-n", "true") .redirectErrorStream(true) .start() .waitFor() != 0 From cc8e9a0fdaea18d740bf94a799a5a5b72de4e09e Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 18:38:09 +0100 Subject: [PATCH 24/50] kommand: recognise wrong sudo passwords reported by sudo-rs --- .../manamiproject/kommand/JavaProcessBuilder.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt index 6b80c8b8..00c09fb6 100644 --- a/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt +++ b/kommand/src/main/kotlin/io/github/manamiproject/kommand/JavaProcessBuilder.kt @@ -7,6 +7,15 @@ import java.io.IOException private const val SUDO = "sudo" +/** + * How sudo reports a wrong password. Classic sudo says "incorrect password attempt", sudo-rs (the default since + * Ubuntu 25.10) says "Authentication failed". + */ +private val INCORRECT_PASSWORD_MESSAGES = setOf( + "incorrect password attempt", + "authentication failed", +) + /** * If you want to execute a command as `sudo` you can pass `command` in [JavaProcessBuilder.executeCmd] either with or * without sudo as first element. It's only important that [CommandLineConfig.useSudo] is set to `true`. If you don't @@ -55,7 +64,7 @@ public class JavaProcessBuilder( output.addAll(process.inputStream.bufferedReader().readLines()) } - if (config.useSudo && output.any { it.lowercase().contains("incorrect password attempt") }) { + if (config.useSudo && output.any { line -> INCORRECT_PASSWORD_MESSAGES.any { line.lowercase().contains(it) } }) { throw IncorrectPasswordAttemptException } From 116884a2334874dc8ea16665a2e056c76fa0d638 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 19:02:53 +0100 Subject: [PATCH 25/50] Fix Gradle 9.6 deprecations and redundant Elvis operators --- anidb/build.gradle.kts | 4 ++-- anilist/build.gradle.kts | 4 ++-- anime-planet/build.gradle.kts | 4 ++-- animenewsnetwork/build.gradle.kts | 4 ++-- anisearch/build.gradle.kts | 4 ++-- core/build.gradle.kts | 4 ++-- .../modb/core/extractor/JsoupCssSelectorDataExtractor.kt | 2 +- .../modb/core/extractor/JsoupXPathDataExtractor.kt | 4 ++-- kitsu/build.gradle.kts | 4 ++-- kommand/build.gradle.kts | 4 ++-- lib/build.gradle.kts | 4 ++-- livechart/build.gradle.kts | 4 ++-- myanimelist/build.gradle.kts | 4 ++-- serde/build.gradle.kts | 4 ++-- simkl/build.gradle.kts | 4 ++-- test/build.gradle.kts | 4 ++-- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/anidb/build.gradle.kts b/anidb/build.gradle.kts index 29edc11a..880b7cfb 100644 --- a/anidb/build.gradle.kts +++ b/anidb/build.gradle.kts @@ -61,12 +61,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anilist/build.gradle.kts b/anilist/build.gradle.kts index b4cb9d96..7b329823 100644 --- a/anilist/build.gradle.kts +++ b/anilist/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anime-planet/build.gradle.kts b/anime-planet/build.gradle.kts index 9e052e0d..898add7d 100644 --- a/anime-planet/build.gradle.kts +++ b/anime-planet/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/animenewsnetwork/build.gradle.kts b/animenewsnetwork/build.gradle.kts index 44f751a7..5b03c8fe 100644 --- a/animenewsnetwork/build.gradle.kts +++ b/animenewsnetwork/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/anisearch/build.gradle.kts b/anisearch/build.gradle.kts index 1c79a220..56c42900 100644 --- a/anisearch/build.gradle.kts +++ b/anisearch/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index a5580f32..c642ccee 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -70,12 +70,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt index 801ec705..e7c0a070 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupCssSelectorDataExtractor.kt @@ -114,7 +114,7 @@ internal object JsoupCssSelectorDataExtractor : DataExtractor { terminatingChild == "text()" -> jsoupElements.eachText() terminatingChild == "node()" -> jsoupElements.dataNodes().map { it.wholeData.trim() } terminatingChild.eitherNullOrBlank() -> jsoupElements.textNodes().map { it.text().trim() }.filter { it.neitherNullNorBlank() } - terminatingChild.startsWith('@') -> jsoupElements.eachAttr(terminatingChild.trimStart('@')) ?: NotFound + terminatingChild.startsWith('@') -> jsoupElements.eachAttr(terminatingChild.trimStart('@')) else -> throw IllegalStateException("unmapped case") } } diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt index e3e40952..32b0b620 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/extractor/JsoupXPathDataExtractor.kt @@ -54,10 +54,10 @@ internal object JsoupXPathDataExtractor : DataExtractor { val value = when { jsoupElements.isEmpty() -> NotFound - entry.third == "text()" -> jsoupElements.eachText() ?: NotFound + entry.third == "text()" -> jsoupElements.eachText() entry.third == "node()" -> jsoupElements.dataNodes().map { it.wholeData.trim() } entry.third.eitherNullOrBlank() -> jsoupElements.textNodes().map { it.text().trim() }.filter { it.neitherNullNorBlank() } - else -> jsoupElements.eachAttr(entry.third) ?: NotFound + else -> jsoupElements.eachAttr(entry.third) } return entry.first to value diff --git a/kitsu/build.gradle.kts b/kitsu/build.gradle.kts index 0fad5e74..0f089d6e 100644 --- a/kitsu/build.gradle.kts +++ b/kitsu/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/kommand/build.gradle.kts b/kommand/build.gradle.kts index 44eec7c2..c0bb65e0 100644 --- a/kommand/build.gradle.kts +++ b/kommand/build.gradle.kts @@ -45,12 +45,12 @@ tasks.withType { maxParallelForks = Runtime.getRuntime().availableProcessors() } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/lib/build.gradle.kts b/lib/build.gradle.kts index 277f7ba7..6e4e8899 100644 --- a/lib/build.gradle.kts +++ b/lib/build.gradle.kts @@ -72,12 +72,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/livechart/build.gradle.kts b/livechart/build.gradle.kts index 1a42b57f..0a0b0fef 100644 --- a/livechart/build.gradle.kts +++ b/livechart/build.gradle.kts @@ -53,12 +53,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/myanimelist/build.gradle.kts b/myanimelist/build.gradle.kts index 22d4b828..ec4d2eec 100644 --- a/myanimelist/build.gradle.kts +++ b/myanimelist/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/serde/build.gradle.kts b/serde/build.gradle.kts index 0dfafeca..719cb11a 100644 --- a/serde/build.gradle.kts +++ b/serde/build.gradle.kts @@ -53,12 +53,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/simkl/build.gradle.kts b/simkl/build.gradle.kts index dd946838..61557106 100644 --- a/simkl/build.gradle.kts +++ b/simkl/build.gradle.kts @@ -51,12 +51,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } diff --git a/test/build.gradle.kts b/test/build.gradle.kts index 53c25b3c..03031117 100644 --- a/test/build.gradle.kts +++ b/test/build.gradle.kts @@ -55,12 +55,12 @@ tasks.withType { systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") } -val sourcesJar by tasks.registering(Jar::class) { +val sourcesJar = tasks.register("sourcesJar") { archiveClassifier.set("sources") from(sourceSets.main.get().allSource) } -val javaDoc by tasks.registering(Jar::class) { +val javaDoc = tasks.register("javaDoc") { archiveClassifier.set("javadoc") from(sourceSets.main.get().allSource) } From 87e74d12629261bbad2fa07f7e6b757355aa2569 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 19:02:54 +0100 Subject: [PATCH 26/50] Shorter message when the AniDB crawler stops because of a ban --- app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 95dd34c6..ceb3dfdd 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -212,7 +212,7 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: block.invoke() } catch (_: AnidbBannedException) { // Expected while a ban lasts, so a stack trace adds nothing. - log.error("Crawler for [$hostname] stopped, because AniDB still answers with [banned] after all retries. Everything downloaded so far is kept. Wait until the ban has been lifted or add [$hostname] to [modb.app.disabledCrawlers].") + log.error("Crawler for [$hostname] stopped, because AniDB still answers with [banned] after all retries. Wait until the ban has been lifted or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { log.error("Crawler for [$hostname] stopped: ${e.message}", e) } From 4b15a2dee19beb2704bbe475973923809acfb3bc Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 22:18:34 +0100 Subject: [PATCH 27/50] Keep reconverted data in DCS entries already updated this week --- .../DefaultDownloadControlStateAccessor.kt | 20 ++++++++- ...DefaultDownloadControlStateAccessorTest.kt | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt index bde6a2e4..d51425f5 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessor.kt @@ -139,8 +139,24 @@ class DefaultDownloadControlStateAccessor( val currentDownloadControlStateEntry = dcsEntry(metaDataProviderConfig, animeId) if (currentDownloadControlStateEntry.lastDownloaded == WeekOfYear.currentWeek()) { - log.debug { "Not updating DCS file for [${animeId}] of [${metaDataProviderConfig.hostname()}], because it has been updated already." } - return false + // A second run in the same week must not count as another week of the schedule. But if the + // raw files have been converted again in the meantime (e.g. by a fixed converter), the data + // of this week has changed and is kept, otherwise the dataset is built from the old version. + if (currentDownloadControlStateEntry.anime == downloadControlStateEntry.anime) { + log.debug { "Not updating DCS file for [${animeId}] of [${metaDataProviderConfig.hostname()}], because it has been updated already." } + return false + } + + log.debug { "Refreshing the data of DCS entry [$animeId] of [${metaDataProviderConfig.hostname()}], which has been updated this week already, without changing its schedule." } + + val refreshed = currentDownloadControlStateEntry.copy(_anime = downloadControlStateEntry.anime) + Json.toJson(refreshed).writeToFile(downloadControlStateFile) + safelyStore(metaDataProviderConfig) { + it.toMutableMap().apply { + put(animeId, refreshed) + } + } + return true } log.info { "Updating DCS entry for [$animeId] of [${metaDataProviderConfig.hostname()}]." } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt index a7061ca4..5d49682b 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateAccessorTest.kt @@ -1162,6 +1162,49 @@ internal class DefaultDownloadControlStateAccessorTest { } } + @Test + fun `refreshes the data of an entry updated this week already, but keeps its schedule`() { + tempDirectory { + // given + val testAppConfig = object: Config by TestAppConfig { + override fun metaDataProviderConfigurations(): Set = setOf(AnilistConfig) + override fun downloadControlStateDirectory(): Directory = tempDir + } + + val firstUpdate = DownloadControlStateEntry( + _weeksWihoutChange = 3, + _lastDownloaded = WeekOfYear.currentWeek(), + _nextDownload = WeekOfYear.currentWeek().plusWeeks(3), + _anime = TestAnimeRawObjects.DefaultAnime.obj, + ) + + val secondUpdate = DownloadControlStateEntry( + _weeksWihoutChange = 4, + _lastDownloaded = WeekOfYear.currentWeek(), + _nextDownload = WeekOfYear.currentWeek().plusWeeks(4), + _anime = TestAnimeRawObjects.AllPropertiesSet.obj, + ) + + tempDir.resolve(AnilistConfig.hostname()).createDirectory() + + val defaultDownloadControlStateAccessor = DefaultDownloadControlStateAccessor( + appConfig = testAppConfig, + mergeLockAccess = TestMergeLockAccessor, + ) + defaultDownloadControlStateAccessor.createOrUpdate(AnilistConfig, "177191", firstUpdate) + + // when + val result = defaultDownloadControlStateAccessor.createOrUpdate(AnilistConfig, "177191", secondUpdate) + + // then + assertThat(result).isTrue() + val stored = defaultDownloadControlStateAccessor.dcsEntry(AnilistConfig, "177191") + assertThat(stored.anime).isEqualTo(TestAnimeRawObjects.AllPropertiesSet.obj) + assertThat(stored.weeksWihoutChange).isEqualTo(3) + assertThat(stored.nextDownload).isEqualTo(WeekOfYear.currentWeek().plusWeeks(3)) + } + } + @Test fun `triggers initialization if necessary`() { tempDirectory { From 473d82001073c663de1f170e6d31eadef021e880 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 22:32:12 +0100 Subject: [PATCH 28/50] Count createOrUpdate calls thread-safely in the changed-id test --- .../DefaultDownloadControlStateUpdaterTest.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt index 71d780eb..33bdf84c 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/downloadcontrolstate/DefaultDownloadControlStateUpdaterTest.kt @@ -30,6 +30,7 @@ import io.github.manamiproject.modb.test.tempDirectory import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested import java.net.URI +import java.util.concurrent.atomic.AtomicInteger import java.time.Clock import java.time.Instant import java.time.ZoneId @@ -741,7 +742,7 @@ internal class DefaultDownloadControlStateUpdaterTest { override fun clock(): Clock = Clock.fixed(Instant.parse("2021-11-02T17:55:43.035Z"), ZoneId.systemDefault()) } - var invocations = 0 + val invocations = AtomicInteger(0) var hasBeenInvoked = false val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { override suspend fun allDcsEntries(metaDataProviderConfig: MetaDataProviderConfig): List = emptyList() @@ -752,7 +753,7 @@ internal class DefaultDownloadControlStateUpdaterTest { } override suspend fun dcsEntryExists(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId): Boolean = false override suspend fun createOrUpdate(metaDataProviderConfig: MetaDataProviderConfig, animeId: AnimeId, downloadControlStateEntry: DownloadControlStateEntry): Boolean { - invocations++ + invocations.incrementAndGet() return true } } @@ -766,7 +767,7 @@ internal class DefaultDownloadControlStateUpdaterTest { downloadControlStateUpdater.updateAll() // then - assertThat(invocations).isEqualTo(2) + assertThat(invocations.get()).isEqualTo(2) assertThat(hasBeenInvoked).isTrue() } } From 4bad65a816e265d57d8054f2affe70fcf2bec029 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 16:15:28 +0000 Subject: [PATCH 29/50] Add the AniDB source setting to the config template Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- config.example.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.example.toml b/config.example.toml index 594e1d22..96410e51 100644 --- a/config.example.toml +++ b/config.example.toml @@ -8,6 +8,8 @@ downloadControlStateDirectory = "" disabledCrawlers = ["anidb.net"] [modb.anidb] +# api | website +source = "api" # Client registered for the AniDB HTTP API. client = "" clientver = "" From 166378ed410b11b696c104c161371cd3643cb0a8 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 18:10:03 +0100 Subject: [PATCH 30/50] Add modb.app.challenge.solverUrl to example config --- config.example.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config.example.toml b/config.example.toml index 96410e51..4dfe72e6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -7,9 +7,23 @@ downloadControlStateDirectory = "" # Crawlers to skip. disabledCrawlers = ["anidb.net"] +[modb.app.challenge] +solverUrl = "http://localhost:8191/v1" + +[modb.app.crawler] +# Dead/pending entries requested again per run, highest IDs first (AniList, Kitsu, MAL, AniDB). +deadEntriesToRecheck = 0 + +[modb.core.httpclient] +# Retries after server errors, 429 and timeouts, for all sites. +maxRetries = 5 + [modb.anidb] # api | website source = "api" # Client registered for the AniDB HTTP API. client = "" clientver = "" +# Retries while AniDB answers "banned", before the AniDB crawler stops. +banRetries = 5 + From 9d6f6ae297873c7c5341145b0abe8b6dc6ecaec3 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Thu, 29 Oct 2026 19:40:19 +0100 Subject: [PATCH 31/50] AniDB: keep entries which the title dump names but the API doesn't serve as pending, recheck them via deadEntriesToRecheck --- .../crawlers/IntegerBasedIdRangeSelector.kt | 4 +- .../modb/app/crawlers/anidb/AnidbCrawler.kt | 53 ++++- .../app/crawlers/anidb/AnidbPendingEntries.kt | 185 ++++++++++++++++++ .../manamiproject/modb/app/TestingAssets.kt | 8 + .../IntegerBasedIdRangeSelectorTest.kt | 102 ++++++++++ .../app/crawlers/anidb/AnidbCrawlerTest.kt | 147 ++++++++++++++ .../anidb/DefaultAnidbPendingEntriesTest.kt | 109 +++++++++++ 7 files changed, 606 insertions(+), 2 deletions(-) create mode 100644 lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt create mode 100644 lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt index cbf28aeb..b01891ab 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelector.kt @@ -28,6 +28,7 @@ import io.github.manamiproject.modb.core.logging.LoggerDelegate * @property downloadControlStateAccessor Access to DCS files. * @property downloadControlStateScheduler Allows to check which anime are scheduled for re-download and which are not. * @property alreadyDownloadedIdsFinder Fetches all IDs which have already been downloaded. + * @property recheckDeadEntries Whether `deadEntriesToRecheck` applies. A crawler which rechecks something else instead turns it off. */ class IntegerBasedIdRangeSelector( private val metaDataProviderConfig: MetaDataProviderConfig, @@ -36,6 +37,7 @@ class IntegerBasedIdRangeSelector( private val downloadControlStateAccessor: DownloadControlStateAccessor = DefaultDownloadControlStateAccessor.instance, private val downloadControlStateScheduler: DownloadControlStateScheduler = DefaultDownloadControlStateScheduler.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, + private val recheckDeadEntries: Boolean = true, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ): IdRangeSelector { @@ -67,7 +69,7 @@ class IntegerBasedIdRangeSelector( // out of the dataset for good. The highest ids are the ones this happens to, because they // are the ones a provider is still filling in, so a slice of them is looked at again. val deadEntries = deadEntriesAccessor.fetchDeadEntries(metaDataProviderConfig).map { it.toInt() }.toSet() - val recheck = deadEntries.sortedDescending().take(deadEntriesToRecheck).toSet() + val recheck = if (recheckDeadEntries) deadEntries.sortedDescending().take(deadEntriesToRecheck).toSet() else emptySet() if (recheck.isNotEmpty()) { log.info { "Rechecking [${recheck.size}] of [${deadEntries.size}] dead entries for [${metaDataProviderConfig.hostname()}]." } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index 4c48f874..e139673f 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -19,6 +19,7 @@ import io.github.manamiproject.modb.app.network.SuspendableHttpClient import io.github.manamiproject.modb.app.network.isRouteFailure import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.DefaultConfigRegistry +import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.IntPropertyDelegate import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.core.coverage.KoverIgnore @@ -43,6 +44,9 @@ import kotlin.time.toDuration * Uses [IntegerBasedIdRangeSelector] to determine which data to download. * Waits a random time between downloads and can move to another connection every few pages, both to reduce pressure * on the metadata provider. + * + * Reads `modb.app.crawler.deadEntriesToRecheck` like the other crawlers, but applies it to the pending entries. The + * AniDB dead entries come from the title dump, so a dead id doesn't exist and asking for it again is pointless. * @since 1.0.0 * @property appConfig Application specific configuration. Uses [AppConfig] by default. * @property metaDataProviderConfig Configuration for a specific metadata provider. @@ -55,6 +59,8 @@ import kotlin.time.toDuration * @property minWaitInMilliseconds Shortest wait before each download. * @property maxWaitInMilliseconds Longest wait before each download. * @property pagesPerConnection Downloads made through one connection before moving to the next. `0` stays on it. + * @property pendingEntries Entries which the title dump names, but the API doesn't serve yet. + * @property titleDump Source of the ids named by the title dump. */ class AnidbCrawler( private val appConfig: Config = AppConfig.instance, @@ -63,6 +69,7 @@ class AnidbCrawler( private val idRangeSelector: IdRangeSelector = IntegerBasedIdRangeSelector( metaDataProviderConfig = metaDataProviderConfig, highestIdDetector = AnidbTitlesDumpHighestIdDetector.instance, + recheckDeadEntries = false, ), private val httpClient: HttpClient = AnidbSource.httpClient(), private val downloader: Downloader = FaultTolerantDownloader( @@ -75,9 +82,19 @@ class AnidbCrawler( isHandledByCaller = { it is CrawlerDetectedException || it is AnidbBannedException }, ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), + private val pendingEntries: AnidbPendingEntries = DefaultAnidbPendingEntries.instance, + private val titleDump: suspend () -> Set = { AnidbTitlesDumpHighestIdDetector.instance.allIds() }, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ): Crawler { + private val deadEntriesToRecheck: Int by IntPropertyDelegate( + configRegistry = configRegistry, + namespace = IntegerBasedIdRangeSelector.CONFIG_NAMESPACE, + default = 0, + ) + + private var idsInTitleDump: Set? = null + private val maxConnectionChanges: Int by IntPropertyDelegate( configRegistry = configRegistry, namespace = CONFIG_NAMESPACE, @@ -132,6 +149,8 @@ class AnidbCrawler( idDownloadList.remove(it.fileName().remove(".${ANIDB_PENDING_FILE_SUFFIX}").toInt()) } + skipPendingEntries(idDownloadList) + when { idDownloadList.isEmpty() -> log.info { "No IDs left for [${metaDataProviderConfig.hostname()}] crawler to download." } else -> startDownload(idDownloadList) @@ -141,6 +160,37 @@ class AnidbCrawler( log.info { "Finished crawling data for [${metaDataProviderConfig.hostname()}]." } } + private suspend fun skipPendingEntries(idDownloadList: MutableList) { + val pending = pendingEntries.all() + + if (pending.isEmpty()) { + return + } + + pendingEntries.removeAllNotIn(titleDumpIds()) + + val stillPending = pendingEntries.all() + val recheck = pendingEntries.selectForRecheck(deadEntriesToRecheck) + idDownloadList.removeAll(stillPending - recheck) + + log.info { "Skipping [${stillPending.size - recheck.size}] pending entries of [${metaDataProviderConfig.hostname()}], rechecking [${recheck.size}]." } + } + + private suspend fun titleDumpIds(): Set = idsInTitleDump ?: titleDump().also { idsInTitleDump = it } + + /** + * An id which the API refuses, but the title dump names, is not dead. Its addition is most likely still pending. + */ + private suspend fun handleNotServed(animeId: AnimeId) { + val id = animeId.toIntOrNull() + + if (id != null && id in titleDumpIds()) { + pendingEntries.add(id) + } else { + deadEntriesAccess.addDeadEntry(animeId, metaDataProviderConfig) + } + } + private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> val animeId = idDownloadList[index] wait() @@ -160,6 +210,7 @@ class AnidbCrawler( val file = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${metaDataProviderConfig.fileSuffix()}") response.writeToFile(file, true) deadEntriesAccess.removeDeadEntry(animeId.toString(), metaDataProviderConfig) + pendingEntries.remove(animeId) } response == ANIDB_PENDING_FILE_INDICATOR -> { val pendingFile = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${ANIDB_PENDING_FILE_SUFFIX}") @@ -182,7 +233,7 @@ class AnidbCrawler( while (true) { try { return downloader.download(animeId.toString()) { - deadEntriesAccess.addDeadEntry(it, metaDataProviderConfig) + handleNotServed(it) } } catch (e: Throwable) { if (e !is CrawlerDetectedException || connectionChanges >= maxConnectionChanges) { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt new file mode 100644 index 00000000..c07f1b8f --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt @@ -0,0 +1,185 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS +import io.github.manamiproject.modb.core.extensions.regularFileExists +import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.time.LocalDate +import kotlin.io.path.readLines +import kotlin.io.path.writeText + +/** + * AniDB entries which the title dump names, but which the API doesn't serve yet, most likely because their addition + * is still pending approval. + * + * Such an id is neither dead nor downloadable. As a dead entry it would leave the dead entries list with every new + * title dump and be requested again on every run, only to be refused again. Kept apart, it is skipped until a recheck + * is asked for, and it leaves the list on its own once it is downloaded or once the title dump no longer names it. + * @since 1.0.0 + */ +interface AnidbPendingEntries { + + /** + * Removes ids which the title dump no longer names. Those are dead entries now. + * @since 1.0.0 + * @param idsInTitleDump All ids named by the current title dump. + */ + suspend fun removeAllNotIn(idsInTitleDump: Set) + + /** + * @since 1.0.0 + * @return All pending ids. + */ + suspend fun all(): Set + + /** + * Selects the pending ids which have been checked longest ago and records today as their last check. + * @since 1.0.0 + * @param numberOfEntries Maximum number of ids to select. + * @return Ids to request again. + */ + suspend fun selectForRecheck(numberOfEntries: Int): Set + + /** + * Adds an id or, if it is pending already, records today as its last check. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun add(id: Int) + + /** + * Removes an id, because it has been downloaded. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun remove(id: Int) +} + +/** + * Keeps the pending entries in the download control state directory, one `id last-check` pair per line. + * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. + */ +class DefaultAnidbPendingEntries( + private val appConfig: Config = AppConfig.instance, +): AnidbPendingEntries { + + private val access = Mutex() + private var entries: MutableMap? = null + + override suspend fun removeAllNotIn(idsInTitleDump: Set) = update { current -> + val removed = current.keys.filterNot { it in idsInTitleDump } + removed.forEach { current.remove(it) } + + if (removed.isNotEmpty()) { + log.info { "Removed [${removed.size}] pending [anidb.net] entries which the title dump no longer names." } + } + } + + override suspend fun all(): Set = access.withLock { load().keys.toSet() } + + override suspend fun selectForRecheck(numberOfEntries: Int): Set { + if (numberOfEntries <= 0) { + return emptySet() + } + + var selected = emptySet() + + update { current -> + selected = current.entries + .sortedWith(compareBy> { it.value }.thenBy { it.key }) + .take(numberOfEntries) + .map { it.key } + .toSet() + + val today = LocalDate.now(appConfig.clock()) + selected.forEach { current[it] = today } + } + + return selected + } + + override suspend fun add(id: Int) = update { current -> + if (id !in current) { + log.info { "[anidb.net] doesn't serve [anidbId=$id] yet, although the title dump names it. Keeping it as pending entry." } + } + + current[id] = LocalDate.now(appConfig.clock()) + } + + override suspend fun remove(id: Int) = update { current -> + if (current.remove(id) != null) { + log.info { "[anidb.net] serves the pending entry [anidbId=$id] now." } + } + } + + private suspend fun update(change: (MutableMap) -> Unit) = access.withLock { + val current = load() + val before = current.toMap() + change(current) + + if (current != before) { + save(current) + } + } + + private suspend fun load(): MutableMap = withContext(LIMITED_FS) { + entries?.let { return@withContext it } + + val file = file() + val loaded = mutableMapOf() + + if (file.regularFileExists()) { + file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith(COMMENT) } + .forEach { line -> + val parts = line.split(' ') + val id = parts[0].toIntOrNull() ?: return@forEach + val lastCheck = parts.getOrNull(1)?.let { runCatching { LocalDate.parse(it) }.getOrNull() } ?: LocalDate.MIN + loaded[id] = lastCheck + } + } + + entries = loaded + return@withContext loaded + } + + private suspend fun save(current: Map) = withContext(LIMITED_FS) { + val file = file() + val content = buildString { + appendLine("$COMMENT AniDB ids which the title dump names, but the API doesn't serve yet. Format: id last-check") + current.toSortedMap().forEach { (id, lastCheck) -> appendLine("$id $lastCheck") } + } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeText(content) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + } + + private fun file() = appConfig.downloadControlStateDirectory().resolve(FILE_NAME) + + companion object { + private val log by LoggerDelegate() + private const val COMMENT = "#" + + /** + * Name of the file in the download control state directory. + * @since 1.0.0 + */ + const val FILE_NAME: String = "anidb-pending-entries.txt" + + /** + * Singleton of [DefaultAnidbPendingEntries]. + * @since 1.0.0 + */ + val instance: DefaultAnidbPendingEntries by lazy { DefaultAnidbPendingEntries() } + } +} diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt index 07665e2c..2827da1f 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt @@ -164,6 +164,14 @@ internal object TestDeadEntriesAccessor: DeadEntriesAccessor { override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = shouldNotBeInvoked() } +internal object TestAnidbPendingEntries: io.github.manamiproject.modb.app.crawlers.anidb.AnidbPendingEntries { + override suspend fun removeAllNotIn(idsInTitleDump: Set) { } + override suspend fun all(): Set = emptySet() + override suspend fun selectForRecheck(numberOfEntries: Int): Set = emptySet() + override suspend fun add(id: Int) { } + override suspend fun remove(id: Int) { } +} + internal object TestReviewedIsolatedEntriesAccessor: ReviewedIsolatedEntriesAccessor { override fun contains(uri: URI): Boolean = shouldNotBeInvoked() override suspend fun addCheckedEntry(uri: URI) = shouldNotBeInvoked() diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt index 21999b4d..17f25ebc 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/IntegerBasedIdRangeSelectorTest.kt @@ -6,6 +6,7 @@ import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateScheduler import io.github.manamiproject.modb.core.config.AnimeId +import io.github.manamiproject.modb.core.config.ConfigRegistry import io.github.manamiproject.modb.core.config.Hostname import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.test.exceptionExpected @@ -246,6 +247,107 @@ internal class IntegerBasedIdRangeSelectorTest { } } + @Test + fun `rechecks the highest dead entries as configured by deadEntriesToRecheck`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + } + + val testHighestIdDetector = object: HighestIdDetector { + override suspend fun detectHighestId(): Int = 8 + } + + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = setOf("3", "5") + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = 5 + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testDefaultAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.app.crawler.deadEntriesToRecheck") 1 else null + } + + val idRangeSelector = IntegerBasedIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + highestIdDetector = testHighestIdDetector, + deadEntriesAccessor = testDeadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testDefaultAlreadyDownloadedIdsFinder, + configRegistry = testConfigRegistry, + ) + + // when + val result = idRangeSelector.idDownloadList() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 2, 4, 5, 6, 7, 8) + } + } + + @Test + fun `doesn't recheck dead entries if the crawler turned it off`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = "example.org" + } + + val testHighestIdDetector = object: HighestIdDetector { + override suspend fun detectHighestId(): Int = 8 + } + + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun fetchDeadEntries(metaDataProviderConfig: MetaDataProviderConfig): Set = setOf("3", "5") + } + + val testDownloadControlStateAccessor = object: DownloadControlStateAccessor by TestDownloadControlStateAccessor { + override suspend fun highestIdAlreadyInDataset(metaDataProviderConfig: MetaDataProviderConfig): Int = 5 + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testDefaultAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = if (key == "modb.app.crawler.deadEntriesToRecheck") 1 else null + } + + val idRangeSelector = IntegerBasedIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + highestIdDetector = testHighestIdDetector, + deadEntriesAccessor = testDeadEntriesAccessor, + downloadControlStateAccessor = testDownloadControlStateAccessor, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testDefaultAlreadyDownloadedIdsFinder, + recheckDeadEntries = false, + configRegistry = testConfigRegistry, + ) + + // when + val result = idRangeSelector.idDownloadList() + + // then + assertThat(result).containsExactlyInAnyOrder(1, 2, 4, 6, 7, 8) + } + } + @Test fun `removes IDs which are not scheduled for current week from generated sequence`() { runTest { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt index 03df50b3..d6f03125 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt @@ -72,6 +72,8 @@ internal class AnidbCrawlerTest { httpClient = testHttpClient, downloader = TestDownloader, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // then @@ -117,6 +119,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -155,6 +159,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -203,6 +209,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -253,6 +261,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = testDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -266,6 +276,129 @@ internal class AnidbCrawlerTest { } } + @Test + fun `keeps an id which the title dump names as pending entry instead of adding it to the dead entries`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1000, 1001) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + onDeadEntry.invoke(id) + return EMPTY + } + } + + val deadEntries = mutableListOf() + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { + deadEntries.add(animeId) + } + } + + val pending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = testDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + titleDump = { setOf(1000) }, + ) + + // when + crawler.start() + + // then + assertThat(pending).containsExactly(1000) + assertThat(deadEntries).containsExactly("1001") + } + } + + @Test + fun `skips pending entries and only requests those selected for a recheck`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1, 2, 3, 4) + } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } + } + + var recheckRequestedFor = -1 + val removed = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun all(): Set = setOf(2, 3) + override suspend fun selectForRecheck(numberOfEntries: Int): Set { + recheckRequestedFor = numberOfEntries + return setOf(3) + } + override suspend fun remove(id: Int) { + removed.add(id) + } + } + + val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry { + override fun int(key: String): Int? = when (key) { + "modb.app.crawler.deadEntriesToRecheck" -> 1 + else -> null + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + titleDump = { setOf(1, 2, 3, 4) }, + configRegistry = testConfigRegistry, + ) + + // when + crawler.start() + + // then + assertThat(recheckRequestedFor).isEqualTo(1) + assertThat(requested).containsExactlyInAnyOrder("1", "3", "4") + assertThat(removed).contains(3) + } + } + @Test fun `directly throws an exception if it's not one of the cases that restart the network controller`() { tempDirectory { @@ -295,6 +428,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -334,6 +469,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -371,6 +508,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -425,6 +564,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -468,6 +609,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -529,6 +672,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = testDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, ) // when @@ -580,6 +725,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = testNetworkController, + pendingEntries = TestAnidbPendingEntries, + titleDump = { emptySet() }, configRegistry = testConfigRegistry, ) } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt new file mode 100644 index 00000000..8b6e9510 --- /dev/null +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbPendingEntriesTest.kt @@ -0,0 +1,109 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.DefaultAnidbPendingEntries.Companion.FILE_NAME +import io.github.manamiproject.modb.core.extensions.Directory +import io.github.manamiproject.modb.test.tempDirectory +import org.assertj.core.api.Assertions.assertThat +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC +import kotlin.io.path.readLines +import kotlin.io.path.writeText +import kotlin.test.Test + +internal class DefaultAnidbPendingEntriesTest { + + private fun testAppConfig(dir: Directory, day: String = "2026-10-29") = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dir + override fun clock(): Clock = Clock.fixed(Instant.parse("${day}T12:00:00Z"), UTC) + } + + @Test + fun `keeps added entries in the download control state directory`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + + // when + pendingEntries.add(20227) + pendingEntries.add(19000) + + // then + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("19000 2026-10-29", "20227 2026-10-29") + assertThat(DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)).all()) + .containsExactlyInAnyOrder(19000, 20227) + } + } + + @Test + fun `removes an entry`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + pendingEntries.add(19000) + + // when + pendingEntries.remove(20227) + + // then + assertThat(pendingEntries.all()).containsExactly(19000) + } + } + + @Test + fun `removes entries which the title dump no longer names`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + pendingEntries.add(19000) + + // when + pendingEntries.removeAllNotIn(setOf(20227, 1)) + + // then + assertThat(pendingEntries.all()).containsExactly(20227) + } + } + + @Test + fun `selects the entries which have been checked longest ago and marks them as checked today`() { + tempDirectory { + // given + tempDir.resolve(FILE_NAME).writeText(""" + # comment + 100 2026-10-20 + 200 2026-10-01 + 300 2026-10-25 + """.trimIndent()) + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + + // when + val result = pendingEntries.selectForRecheck(2) + + // then + assertThat(result).containsExactlyInAnyOrder(100, 200) + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("100 2026-10-29", "200 2026-10-29", "300 2026-10-25") + } + } + + @Test + fun `selects nothing if no recheck is configured`() { + tempDirectory { + // given + val pendingEntries = DefaultAnidbPendingEntries(appConfig = testAppConfig(tempDir)) + pendingEntries.add(20227) + + // when + val result = pendingEntries.selectForRecheck(0) + + // then + assertThat(result).isEmpty() + } + } +} From 0cde8c1d31936560244b7b3a071cdff45025e15a Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 17:29:43 +0000 Subject: [PATCH 32/50] AniDB website: hentai and removed entries are dead, not pending, even though the title dump names them Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/app/crawlers/anidb/AnidbCrawler.kt | 7 ++- .../app/crawlers/anidb/AnidbCrawlerTest.kt | 63 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index e139673f..1e6649a0 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -5,6 +5,7 @@ import io.github.manamiproject.modb.anidb.AnidbApiDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR import io.github.manamiproject.modb.anidb.AnidbBannedException +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config @@ -180,13 +181,17 @@ class AnidbCrawler( /** * An id which the API refuses, but the title dump names, is not dead. Its addition is most likely still pending. + * The website only reports an entry as not served when the page says why (hentai, removed from anidb). That is + * final, even though the title dump still names hentai, so it always becomes a dead entry. */ private suspend fun handleNotServed(animeId: AnimeId) { val id = animeId.toIntOrNull() + val fromWebsite = metaDataProviderConfig.fileSuffix() == AnidbWebsiteConfig.fileSuffix() - if (id != null && id in titleDumpIds()) { + if (!fromWebsite && id != null && id in titleDumpIds()) { pendingEntries.add(id) } else { + id?.let { pendingEntries.remove(it) } deadEntriesAccess.addDeadEntry(animeId, metaDataProviderConfig) } } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt index d6f03125..25954ede 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt @@ -1,6 +1,7 @@ package io.github.manamiproject.modb.app.crawlers.anidb import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.* @@ -333,6 +334,68 @@ internal class AnidbCrawlerTest { } } + @Test + fun `on the website, hentai and removed entries are dead even though the title dump names them`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(1000, 1001) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + onDeadEntry.invoke(id) + return EMPTY + } + } + + val deadEntries = mutableListOf() + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun addDeadEntry(animeId: AnimeId, metaDataProviderConfig: MetaDataProviderConfig) { + deadEntries.add(animeId) + } + } + + val added = mutableListOf() + val removed = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + added.add(id) + } + override suspend fun remove(id: Int) { + removed.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = testDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + titleDump = { setOf(1000) }, + ) + + // when + crawler.start() + + // then + assertThat(added).isEmpty() + assertThat(removed).containsExactly(1000, 1001) + assertThat(deadEntries).containsExactly("1000", "1001") + } + } + @Test fun `skips pending entries and only requests those selected for a recheck`() { tempDirectory { From be4c3440437784816b84f38a06cf6b80d6c483a4 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 17:49:57 +0000 Subject: [PATCH 33/50] AniDB website: skip 18+ restricted entries instead of marking them dead Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../io/github/manamiproject/modb/anidb/AnidbDownloader.kt | 7 ++++++- .../github/manamiproject/modb/anidb/AnidbDownloaderTest.kt | 4 ++-- .../manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt | 4 ++-- .../modb/app/crawlers/anidb/AnidbCrawlerTest.kt | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt index 7ea1bc5f..e2093fad 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt @@ -39,7 +39,12 @@ public class AnidbDownloader( check(response.isOk() || response.code == 404) { "Unexpected response code [anidbId=$id], [responseCode=${response.code}]" } return@withContext when { - responseChecker.isHentai() || responseChecker.isRemovedFromAnidb() -> { + // 18+ pages show no data without an account: skipped, neither dead nor pending + responseChecker.isHentai() -> { + log.info { "Skipping [anidbId=$id], because it is 18+ restricted." } + EMPTY + } + responseChecker.isRemovedFromAnidb() -> { log.info { "Adding [anidbId=$id] to dead-entries list" } onDeadEntry.invoke(id) EMPTY diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt index 40c5e60d..28990bdb 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt @@ -156,7 +156,7 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM } @Test - fun `invoke onDeadEntry if the response body indicates a hentai`() { + fun `skips an 18+ restricted entry without invoking onDeadEntry`() { runTest { // given val id = "11376" @@ -189,7 +189,7 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM // then assertThat(result).isEmpty() - assertThat(deadEntry).isEqualTo(id) + assertThat(deadEntry).isEmpty() } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index 1e6649a0..4aeafc47 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -181,8 +181,8 @@ class AnidbCrawler( /** * An id which the API refuses, but the title dump names, is not dead. Its addition is most likely still pending. - * The website only reports an entry as not served when the page says why (hentai, removed from anidb). That is - * final, even though the title dump still names hentai, so it always becomes a dead entry. + * The website only reports an entry as not served when the page says it has been removed from anidb. That is + * final, so it always becomes a dead entry. (18+ entries are skipped by the downloader, they never get here.) */ private suspend fun handleNotServed(animeId: AnimeId) { val id = animeId.toIntOrNull() diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt index 25954ede..ae2156ae 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt @@ -335,7 +335,7 @@ internal class AnidbCrawlerTest { } @Test - fun `on the website, hentai and removed entries are dead even though the title dump names them`() { + fun `on the website, an entry reported as removed is dead even though the title dump names it`() { tempDirectory { // given val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { From ed80469fea933694e001d325a54853464c3e758b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 18:39:44 +0000 Subject: [PATCH 34/50] AniDB: derive dead entries from the title dump in website mode too The title dump replaced the dead entries under AnidbConfig, which isn't the supported config in website mode, so the replacement was silently skipped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../anidb/AnidbTitlesDumpHighestIdDetector.kt | 10 +++--- .../AnidbTitlesDumpHighestIdDetectorTest.kt | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt index a0cafbd1..4039ee79 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetector.kt @@ -1,6 +1,5 @@ package io.github.manamiproject.modb.app.crawlers.anidb -import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.crawlers.HighestIdDetector @@ -8,6 +7,7 @@ import io.github.manamiproject.modb.app.dataset.DeadEntriesAccessor import io.github.manamiproject.modb.app.dataset.DefaultDeadEntriesAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DefaultDownloadControlStateAccessor import io.github.manamiproject.modb.app.downloadcontrolstate.DownloadControlStateAccessor +import io.github.manamiproject.modb.core.config.MetaDataProviderConfig import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_NETWORK import io.github.manamiproject.modb.core.extensions.RegularFile import io.github.manamiproject.modb.core.extensions.regularFileExists @@ -49,6 +49,7 @@ import kotlin.io.path.writeBytes * @property deadEntriesAccessor Access to dead entries files. * @property downloadControlStateAccessor Access to DCS files. * @property maxAge Age after which the kept dump is downloaded again. + * @property metaDataProviderConfig Config of the configured AniDB source, under which the dead entries are kept. */ class AnidbTitlesDumpHighestIdDetector( private val appConfig: Config = AppConfig.instance, @@ -57,6 +58,7 @@ class AnidbTitlesDumpHighestIdDetector( private val deadEntriesAccessor: DeadEntriesAccessor = DefaultDeadEntriesAccessor.instance, private val downloadControlStateAccessor: DownloadControlStateAccessor = DefaultDownloadControlStateAccessor.instance, private val maxAge: Duration = Duration.ofDays(1), + private val metaDataProviderConfig: MetaDataProviderConfig = AnidbSource.config(), ): HighestIdDetector { private val dumpAccess = Mutex() @@ -123,10 +125,10 @@ class AnidbTitlesDumpHighestIdDetector( private suspend fun updateDeadEntries(ids: Set, highestId: Int) { // Each dead entry removes the corresponding DCS file. If the dump names fewer anime than // the dataset already has, it is more likely to be broken than AniDB having deleted them. - val highestIdAlreadyInDataset = downloadControlStateAccessor.highestIdAlreadyInDataset(AnidbConfig) + val highestIdAlreadyInDataset = downloadControlStateAccessor.highestIdAlreadyInDataset(metaDataProviderConfig) if (highestId < highestIdAlreadyInDataset) { - log.warn { "Not updating the [${AnidbConfig.hostname()}] dead entries, because the highest id of the title dump [$highestId] is smaller than the highest id already in the dataset [$highestIdAlreadyInDataset]." } + log.warn { "Not updating the [${metaDataProviderConfig.hostname()}] dead entries, because the highest id of the title dump [$highestId] is smaller than the highest id already in the dataset [$highestIdAlreadyInDataset]." } return } @@ -135,7 +137,7 @@ class AnidbTitlesDumpHighestIdDetector( .map { it.toString() } .toSet() - deadEntriesAccessor.replaceDeadEntries(deadEntries, AnidbConfig) + deadEntriesAccessor.replaceDeadEntries(deadEntries, metaDataProviderConfig) } private fun format(instant: Instant): String = TIMESTAMP_FORMAT.format(instant.atZone(appConfig.clock().zone)) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt index 7c025063..e3e2f319 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbTitlesDumpHighestIdDetectorTest.kt @@ -1,6 +1,7 @@ package io.github.manamiproject.modb.app.crawlers.anidb import io.github.manamiproject.modb.anidb.AnidbConfig +import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.app.TestAppConfig import io.github.manamiproject.modb.app.TestDeadEntriesAccessor import io.github.manamiproject.modb.app.TestDownloadControlStateAccessor @@ -120,6 +121,35 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { } } + @Test + fun `keeps the dead entries under the website config when the website is the source`() { + tempDirectory { + // given + var replacedWith = emptySet() + var replacedFor: MetaDataProviderConfig? = null + val testDeadEntriesAccessor = object: DeadEntriesAccessor by TestDeadEntriesAccessor { + override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { + replacedWith = animeIds + replacedFor = metaDataProviderConfig + } + } + + val detector = detector( + tempDir = tempDir, + httpClient = clientReturning(smallDump), + deadEntriesAccessor = testDeadEntriesAccessor, + metaDataProviderConfig = AnidbWebsiteConfig, + ) + + // when + detector.detectHighestId() + + // then + assertThat(replacedWith).containsExactlyInAnyOrder("2", "4") + assertThat(replacedFor).isEqualTo(AnidbWebsiteConfig) + } + } + @Test fun `doesn't touch the dead entries if the dump ends below the highest id already in the dataset`() { tempDirectory { @@ -260,6 +290,7 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { override suspend fun replaceDeadEntries(animeIds: Set, metaDataProviderConfig: MetaDataProviderConfig) { } }, highestIdAlreadyInDataset: Int = 0, + metaDataProviderConfig: MetaDataProviderConfig = AnidbConfig, ): AnidbTitlesDumpHighestIdDetector { val testAppConfig = object: Config by TestAppConfig { override fun downloadControlStateDirectory(): Directory = tempDir @@ -275,6 +306,7 @@ internal class AnidbTitlesDumpHighestIdDetectorTest { httpClient = httpClient, deadEntriesAccessor = deadEntriesAccessor, downloadControlStateAccessor = testDownloadControlStateAccessor, + metaDataProviderConfig = metaDataProviderConfig, ) } From a0a892f232f5ffc0f143a96623211a608dc59e7b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 19:23:59 +0000 Subject: [PATCH 35/50] AniDB: keep pending entries of both modes in the pending list, skip 18+ entries via a list in website mode The website no longer writes .pending files. A pending entry goes to anidb-pending-entries.txt and is rechecked through deadEntriesToRecheck, like in API mode. Existing .pending files of the current week are moved there. 18+ entries are kept in anidb-restricted-entries.txt and skipped on every website run, until the title dump no longer names them. API mode ignores the list, because the API serves them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/anidb/AnidbDownloader.kt | 11 +- .../modb/anidb/AnidbDownloaderTest.kt | 5 +- config.example.toml | 2 +- .../modb/app/crawlers/anidb/AnidbCrawler.kt | 56 +++-- .../app/crawlers/anidb/AnidbPendingEntries.kt | 8 +- .../crawlers/anidb/AnidbRestrictedEntries.kt | 144 ++++++++++++ .../manamiproject/modb/app/TestingAssets.kt | 7 + .../app/crawlers/anidb/AnidbCrawlerTest.kt | 208 +++++++++++++++++- .../DefaultAnidbRestrictedEntriesTest.kt | 67 ++++++ 9 files changed, 478 insertions(+), 30 deletions(-) create mode 100644 lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt create mode 100644 lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt index e2093fad..fcf15e64 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloader.kt @@ -14,6 +14,7 @@ import kotlinx.coroutines.withContext /** * Downloads anime data from anidb.net * In case the anime addition is pending, the [download] function will return a static [String] (see [ANIDB_PENDING_FILE_INDICATOR]). + * In case the anime is 18+ restricted, it returns [ANIDB_RESTRICTED_INDICATOR] instead. * The caller must handle this behavior. * @since 1.0.0 * @property metaDataProviderConfig Configuration for downloading data. @@ -42,7 +43,7 @@ public class AnidbDownloader( // 18+ pages show no data without an account: skipped, neither dead nor pending responseChecker.isHentai() -> { log.info { "Skipping [anidbId=$id], because it is 18+ restricted." } - EMPTY + ANIDB_RESTRICTED_INDICATOR } responseChecker.isRemovedFromAnidb() -> { log.info { "Adding [anidbId=$id] to dead-entries list" } @@ -64,9 +65,15 @@ public class AnidbDownloader( public val instance: AnidbDownloader by lazy { AnidbDownloader() } /** - * Indicator for pending files which have been downloaded, but cannot be converted. + * Returned for an entry whose addition is still pending. There is nothing to convert. * @since 7.0.0 */ public const val ANIDB_PENDING_FILE_INDICATOR: String = ">pending<" + + /** + * Returned for an entry which is 18+ restricted. The page shows no data without an account. + * @since 7.0.0 + */ + public const val ANIDB_RESTRICTED_INDICATOR: String = ">restricted<" } } \ No newline at end of file diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt index 28990bdb..3b130d7a 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbDownloaderTest.kt @@ -3,6 +3,7 @@ package io.github.manamiproject.modb.anidb import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.WireMock.* import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR import io.github.manamiproject.modb.core.config.AnimeId import io.github.manamiproject.modb.core.config.FileSuffix import io.github.manamiproject.modb.core.config.Hostname @@ -156,7 +157,7 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM } @Test - fun `skips an 18+ restricted entry without invoking onDeadEntry`() { + fun `returns the restricted indicator for an 18+ entry without invoking onDeadEntry`() { runTest { // given val id = "11376" @@ -188,7 +189,7 @@ internal class AnidbDownloaderTest : MockServerTestCase by WireM } // then - assertThat(result).isEmpty() + assertThat(result).isEqualTo(ANIDB_RESTRICTED_INDICATOR) assertThat(deadEntry).isEmpty() } } diff --git a/config.example.toml b/config.example.toml index 4dfe72e6..13e1f27b 100644 --- a/config.example.toml +++ b/config.example.toml @@ -11,7 +11,7 @@ disabledCrawlers = ["anidb.net"] solverUrl = "http://localhost:8191/v1" [modb.app.crawler] -# Dead/pending entries requested again per run, highest IDs first (AniList, Kitsu, MAL, AniDB). +# Requested again per run: dead entries of AniList, Kitsu, MAL (highest IDs first), pending entries of AniDB (longest unchecked first). deadEntriesToRecheck = 0 [modb.core.httpclient] diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index 4aeafc47..224563d9 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -4,6 +4,7 @@ import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anidb.AnidbApiDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR import io.github.manamiproject.modb.anidb.AnidbBannedException import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.CrawlerDetectedException @@ -36,6 +37,7 @@ import kotlinx.coroutines.delay import java.net.ConnectException import java.net.NoRouteToHostException import java.net.UnknownHostException +import kotlin.io.path.deleteIfExists import kotlin.time.DurationUnit.MILLISECONDS import kotlin.time.DurationUnit.SECONDS import kotlin.time.toDuration @@ -60,7 +62,8 @@ import kotlin.time.toDuration * @property minWaitInMilliseconds Shortest wait before each download. * @property maxWaitInMilliseconds Longest wait before each download. * @property pagesPerConnection Downloads made through one connection before moving to the next. `0` stays on it. - * @property pendingEntries Entries which the title dump names, but the API doesn't serve yet. + * @property pendingEntries Entries whose addition is most likely still pending, in both modes. + * @property restrictedEntries 18+ entries, skipped in website mode. * @property titleDump Source of the ids named by the title dump. */ class AnidbCrawler( @@ -84,6 +87,7 @@ class AnidbCrawler( ), private val networkController: NetworkController = NetworkControllers.forDeployment(hostname = metaDataProviderConfig.hostname()), private val pendingEntries: AnidbPendingEntries = DefaultAnidbPendingEntries.instance, + private val restrictedEntries: AnidbRestrictedEntries = DefaultAnidbRestrictedEntries.instance, private val titleDump: suspend () -> Set = { AnidbTitlesDumpHighestIdDetector.instance.allIds() }, configRegistry: ConfigRegistry = DefaultConfigRegistry.instance, ): Crawler { @@ -146,10 +150,8 @@ class AnidbCrawler( val idDownloadList = idRangeSelector.idDownloadList().toMutableList() if (idDownloadList.isNotEmpty()) { - appConfig.workingDir(metaDataProviderConfig).listRegularFiles("*.${ANIDB_PENDING_FILE_SUFFIX}").forEach { - idDownloadList.remove(it.fileName().remove(".${ANIDB_PENDING_FILE_SUFFIX}").toInt()) - } - + movePendingFilesToPendingEntries() + skipRestrictedEntries(idDownloadList) skipPendingEntries(idDownloadList) when { @@ -161,6 +163,31 @@ class AnidbCrawler( log.info { "Finished crawling data for [${metaDataProviderConfig.hostname()}]." } } + /** + * Earlier versions kept a pending website entry as a `.pending` file in the week's working directory. + */ + private suspend fun movePendingFilesToPendingEntries() { + appConfig.workingDir(metaDataProviderConfig).listRegularFiles("*.${ANIDB_PENDING_FILE_SUFFIX}").forEach { file -> + file.fileName().remove(".${ANIDB_PENDING_FILE_SUFFIX}").toIntOrNull()?.let { pendingEntries.add(it) } + file.deleteIfExists() + } + } + + private suspend fun skipRestrictedEntries(idDownloadList: MutableList) { + if (!isWebsite() || restrictedEntries.all().isEmpty()) { + return + } + + restrictedEntries.removeAllNotIn(titleDumpIds()) + + val restricted = restrictedEntries.all() + idDownloadList.removeAll(restricted) + + log.info { "Skipping [${restricted.size}] 18+ entries of [${metaDataProviderConfig.hostname()}]." } + } + + private fun isWebsite(): Boolean = metaDataProviderConfig.fileSuffix() == AnidbWebsiteConfig.fileSuffix() + private suspend fun skipPendingEntries(idDownloadList: MutableList) { val pending = pendingEntries.all() @@ -182,13 +209,12 @@ class AnidbCrawler( /** * An id which the API refuses, but the title dump names, is not dead. Its addition is most likely still pending. * The website only reports an entry as not served when the page says it has been removed from anidb. That is - * final, so it always becomes a dead entry. (18+ entries are skipped by the downloader, they never get here.) + * final, so it always becomes a dead entry. (Pending and 18+ entries are answered with an indicator instead.) */ private suspend fun handleNotServed(animeId: AnimeId) { val id = animeId.toIntOrNull() - val fromWebsite = metaDataProviderConfig.fileSuffix() == AnidbWebsiteConfig.fileSuffix() - if (!fromWebsite && id != null && id in titleDumpIds()) { + if (!isWebsite() && id != null && id in titleDumpIds()) { pendingEntries.add(id) } else { id?.let { pendingEntries.remove(it) } @@ -211,15 +237,17 @@ class AnidbCrawler( val response = downloadChangingConnectionWhenRefused(animeId) when { - response.neitherNullNorBlank() && response != ANIDB_PENDING_FILE_INDICATOR -> { + response == ANIDB_PENDING_FILE_INDICATOR -> pendingEntries.add(animeId) + response == ANIDB_RESTRICTED_INDICATOR -> { + pendingEntries.remove(animeId) + restrictedEntries.add(animeId) + } + response.neitherNullNorBlank() -> { val file = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${metaDataProviderConfig.fileSuffix()}") response.writeToFile(file, true) deadEntriesAccess.removeDeadEntry(animeId.toString(), metaDataProviderConfig) pendingEntries.remove(animeId) - } - response == ANIDB_PENDING_FILE_INDICATOR -> { - val pendingFile = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${ANIDB_PENDING_FILE_SUFFIX}") - response.writeToFile(pendingFile, true) + restrictedEntries.remove(animeId) } } } @@ -284,7 +312,7 @@ class AnidbCrawler( val instance: AnidbCrawler by lazy { AnidbCrawler() } /** - * Suffix for pending files which have been downloaded, but cannot be converted. + * Suffix of the pending files which earlier versions wrote. They are moved to the pending entries on start. * @since 7.0.0 */ const val ANIDB_PENDING_FILE_SUFFIX: String = "pending" diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt index c07f1b8f..5a373fd0 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbPendingEntries.kt @@ -16,8 +16,8 @@ import kotlin.io.path.readLines import kotlin.io.path.writeText /** - * AniDB entries which the title dump names, but which the API doesn't serve yet, most likely because their addition - * is still pending approval. + * AniDB entries whose addition is still pending approval. The website says so, the API only refuses them, while the + * title dump already names them. * * Such an id is neither dead nor downloadable. As a dead entry it would leave the dead entries list with every new * title dump and be requested again on every run, only to be refused again. Kept apart, it is skipped until a recheck @@ -108,7 +108,7 @@ class DefaultAnidbPendingEntries( override suspend fun add(id: Int) = update { current -> if (id !in current) { - log.info { "[anidb.net] doesn't serve [anidbId=$id] yet, although the title dump names it. Keeping it as pending entry." } + log.info { "Keeping [anidbId=$id] as pending entry, its addition is most likely not approved yet." } } current[id] = LocalDate.now(appConfig.clock()) @@ -155,7 +155,7 @@ class DefaultAnidbPendingEntries( private suspend fun save(current: Map) = withContext(LIMITED_FS) { val file = file() val content = buildString { - appendLine("$COMMENT AniDB ids which the title dump names, but the API doesn't serve yet. Format: id last-check") + appendLine("$COMMENT AniDB ids whose addition is still pending. Format: id last-check") current.toSortedMap().forEach { (id, lastCheck) -> appendLine("$id $lastCheck") } } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt new file mode 100644 index 00000000..fea9e070 --- /dev/null +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbRestrictedEntries.kt @@ -0,0 +1,144 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.config.AppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS +import io.github.manamiproject.modb.core.extensions.regularFileExists +import io.github.manamiproject.modb.core.logging.LoggerDelegate +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.StandardCopyOption.ATOMIC_MOVE +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import kotlin.io.path.readLines +import kotlin.io.path.writeText + +/** + * AniDB entries which are 18+ restricted. The website shows no data for them without an account, so in website mode + * they are skipped on every run. The API serves them, so they don't apply in API mode. + * @since 1.0.0 + */ +interface AnidbRestrictedEntries { + + /** + * Removes ids which the title dump no longer names. Those are dead entries now. + * @since 1.0.0 + * @param idsInTitleDump All ids named by the current title dump. + */ + suspend fun removeAllNotIn(idsInTitleDump: Set) + + /** + * @since 1.0.0 + * @return All restricted ids. + */ + suspend fun all(): Set + + /** + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun add(id: Int) + + /** + * Removes an id, because it has been downloaded. + * @since 1.0.0 + * @param id AniDB id. + */ + suspend fun remove(id: Int) +} + +/** + * Keeps the restricted entries in the download control state directory, one id per line. + * @since 1.0.0 + * @property appConfig Application specific configuration. Uses [AppConfig] by default. + */ +class DefaultAnidbRestrictedEntries( + private val appConfig: Config = AppConfig.instance, +): AnidbRestrictedEntries { + + private val access = Mutex() + private var entries: MutableSet? = null + + override suspend fun removeAllNotIn(idsInTitleDump: Set) = update { current -> + val removed = current.filterNot { it in idsInTitleDump } + current.removeAll(removed.toSet()) + + if (removed.isNotEmpty()) { + log.info { "Removed [${removed.size}] 18+ [anidb.net] entries which the title dump no longer names." } + } + } + + override suspend fun all(): Set = access.withLock { load().toSet() } + + override suspend fun add(id: Int) = update { current -> + if (current.add(id)) { + log.info { "Keeping [anidbId=$id] as 18+ entry, it is skipped on the website from now on." } + } + } + + override suspend fun remove(id: Int) = update { current -> + if (current.remove(id)) { + log.info { "[anidb.net] serves the 18+ entry [anidbId=$id] now." } + } + } + + private suspend fun update(change: (MutableSet) -> Unit) = access.withLock { + val current = load() + val before = current.toSet() + change(current) + + if (current != before) { + save(current) + } + } + + private suspend fun load(): MutableSet = withContext(LIMITED_FS) { + entries?.let { return@withContext it } + + val file = file() + val loaded = mutableSetOf() + + if (file.regularFileExists()) { + file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith(COMMENT) } + .mapNotNull { it.toIntOrNull() } + .forEach { loaded.add(it) } + } + + entries = loaded + return@withContext loaded + } + + private suspend fun save(current: Set) = withContext(LIMITED_FS) { + val file = file() + val content = buildString { + appendLine("$COMMENT AniDB ids which are 18+ restricted and skipped in website mode.") + current.sorted().forEach { appendLine(it.toString()) } + } + + val tempFile = file.resolveSibling("${file.fileName}.tmp") + tempFile.writeText(content) + Files.move(tempFile, file, REPLACE_EXISTING, ATOMIC_MOVE) + } + + private fun file() = appConfig.downloadControlStateDirectory().resolve(FILE_NAME) + + companion object { + private val log by LoggerDelegate() + private const val COMMENT = "#" + + /** + * Name of the file in the download control state directory. + * @since 1.0.0 + */ + const val FILE_NAME: String = "anidb-restricted-entries.txt" + + /** + * Singleton of [DefaultAnidbRestrictedEntries]. + * @since 1.0.0 + */ + val instance: DefaultAnidbRestrictedEntries by lazy { DefaultAnidbRestrictedEntries() } + } +} diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt index 2827da1f..97d977a2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/TestingAssets.kt @@ -172,6 +172,13 @@ internal object TestAnidbPendingEntries: io.github.manamiproject.modb.app.crawle override suspend fun remove(id: Int) { } } +internal object TestAnidbRestrictedEntries: io.github.manamiproject.modb.app.crawlers.anidb.AnidbRestrictedEntries { + override suspend fun removeAllNotIn(idsInTitleDump: Set) { } + override suspend fun all(): Set = emptySet() + override suspend fun add(id: Int) { } + override suspend fun remove(id: Int) { } +} + internal object TestReviewedIsolatedEntriesAccessor: ReviewedIsolatedEntriesAccessor { override fun contains(uri: URI): Boolean = shouldNotBeInvoked() override suspend fun addCheckedEntry(uri: URI) = shouldNotBeInvoked() diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt index ae2156ae..628e7f5b 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawlerTest.kt @@ -3,6 +3,7 @@ package io.github.manamiproject.modb.app.crawlers.anidb import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anidb.AnidbWebsiteConfig import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_PENDING_FILE_INDICATOR +import io.github.manamiproject.modb.anidb.AnidbDownloader.Companion.ANIDB_RESTRICTED_INDICATOR import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.app.* import io.github.manamiproject.modb.app.config.Config @@ -74,6 +75,7 @@ internal class AnidbCrawlerTest { downloader = TestDownloader, networkController = testNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -121,6 +123,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -161,6 +164,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -176,7 +180,7 @@ internal class AnidbCrawlerTest { } @Test - fun `removes IDs of anime which are pending addition and have been downloaded already`() { + fun `moves pending files of earlier versions to the pending entries and skips them`() { tempDirectory { // given val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { @@ -203,6 +207,14 @@ internal class AnidbCrawlerTest { ANIDB_PENDING_FILE_INDICATOR.writeToFile(tempDir.resolve("1254.${ANIDB_PENDING_FILE_SUFFIX}")) + val pending = mutableSetOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + override suspend fun all(): Set = pending.toSet() + } + val crawler = AnidbCrawler( appConfig = testAppConfig, metaDataProviderConfig = testMetaDataProviderConfig, @@ -210,18 +222,21 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, - pendingEntries = TestAnidbPendingEntries, - titleDump = { emptySet() }, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, + titleDump = { setOf(1254) }, ) // when crawler.start() // then + assertThat(pending).containsExactly(1254) assertThat(tempDir).isDirectoryContaining { it.fileName() == "1535.xml" } assertThat(tempDir).isDirectoryContaining { it.fileName() == "23.xml" } assertThat(tempDir).isDirectoryContaining { it.fileName() == "424.xml" } assertThat(tempDir).isDirectoryNotContaining { it.fileName() == "1254.xml" } + assertThat(tempDir).isDirectoryNotContaining { it.fileName() == "1254.${ANIDB_PENDING_FILE_SUFFIX}" } } } @@ -263,6 +278,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -322,6 +338,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { setOf(1000) }, ) @@ -383,6 +400,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { setOf(1000) }, ) @@ -448,6 +466,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { setOf(1, 2, 3, 4) }, configRegistry = testConfigRegistry, ) @@ -492,6 +511,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -533,6 +553,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -545,7 +566,7 @@ internal class AnidbCrawlerTest { } @Test - fun `creates a file which indicates that the respective anime is pending addition and won't require another download`() { + fun `adds an entry whose addition is pending to the pending entries instead of creating a file`() { tempDirectory { // given val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { @@ -564,6 +585,13 @@ internal class AnidbCrawlerTest { override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = ANIDB_PENDING_FILE_INDICATOR } + val pending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun add(id: Int) { + pending.add(id) + } + } + val crawler = AnidbCrawler( appConfig = testAppConfig, metaDataProviderConfig = testMetaDataProviderConfig, @@ -571,7 +599,8 @@ internal class AnidbCrawlerTest { deadEntriesAccess = TestDeadEntriesAccessor, idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, - pendingEntries = TestAnidbPendingEntries, + pendingEntries = testPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -579,9 +608,170 @@ internal class AnidbCrawlerTest { crawler.start() // then - assertThat(tempDir).isDirectoryContaining { file -> - file.fileName() == "1000.pending" + assertThat(pending).containsExactly(1000) + assertThat(tempDir.listDirectoryEntries()).isEmpty() + } + } + + @Test + fun `on the website, keeps an 18+ entry in the restricted entries`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531) + } + + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = ANIDB_RESTRICTED_INDICATOR + } + + val removedFromPending = mutableListOf() + val testPendingEntries = object: AnidbPendingEntries by TestAnidbPendingEntries { + override suspend fun remove(id: Int) { + removedFromPending.add(id) + } + } + + val restricted = mutableListOf() + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun add(id: Int) { + restricted.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = testPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531) }, + ) + + // when + crawler.start() + + // then + assertThat(restricted).containsExactly(3531) + assertThat(removedFromPending).containsExactly(3531) + assertThat(tempDir.listDirectoryEntries()).isEmpty() + } + } + + @Test + fun `on the website, skips the restricted entries which the title dump still names`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbWebsiteConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531, 23) + } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } + } + + val restricted = mutableSetOf(3531, 999) + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun all(): Set = restricted.toSet() + override suspend fun removeAllNotIn(idsInTitleDump: Set) { + restricted.retainAll(idsInTitleDump) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531, 23) }, + ) + + // when + crawler.start() + + // then + assertThat(requested).containsExactly("23") + assertThat(restricted).containsExactly(3531) + } + } + + @Test + fun `in API mode, requests restricted entries like any other`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by AnidbConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object : Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testIdRangeSelector = object: IdRangeSelector { + override suspend fun idDownloadList(): List = listOf(3531) } + + val requested = mutableListOf() + val testAnimeDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + requested.add(id) + return "content $id" + } + } + + val removedFromRestricted = mutableListOf() + val testRestrictedEntries = object: AnidbRestrictedEntries by TestAnidbRestrictedEntries { + override suspend fun all(): Set = setOf(3531) + override suspend fun remove(id: Int) { + removedFromRestricted.add(id) + } + } + + val crawler = AnidbCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloader = testAnimeDownloader, + deadEntriesAccess = TestDeadEntriesAccessor, + idRangeSelector = testIdRangeSelector, + networkController = TestNetworkController, + pendingEntries = TestAnidbPendingEntries, + restrictedEntries = testRestrictedEntries, + titleDump = { setOf(3531) }, + ) + + // when + crawler.start() + + // then + assertThat(requested).containsExactly("3531") + assertThat(removedFromRestricted).containsExactly(3531) } } @@ -628,6 +818,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = testNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -673,6 +864,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = TestNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -736,6 +928,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = testNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, ) @@ -789,6 +982,7 @@ internal class AnidbCrawlerTest { idRangeSelector = testIdRangeSelector, networkController = testNetworkController, pendingEntries = TestAnidbPendingEntries, + restrictedEntries = TestAnidbRestrictedEntries, titleDump = { emptySet() }, configRegistry = testConfigRegistry, ) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt new file mode 100644 index 00000000..c96238bb --- /dev/null +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/DefaultAnidbRestrictedEntriesTest.kt @@ -0,0 +1,67 @@ +package io.github.manamiproject.modb.app.crawlers.anidb + +import io.github.manamiproject.modb.app.TestAppConfig +import io.github.manamiproject.modb.app.config.Config +import io.github.manamiproject.modb.app.crawlers.anidb.DefaultAnidbRestrictedEntries.Companion.FILE_NAME +import io.github.manamiproject.modb.core.extensions.Directory +import io.github.manamiproject.modb.test.tempDirectory +import org.assertj.core.api.Assertions.assertThat +import kotlin.io.path.readLines +import kotlin.test.Test + +internal class DefaultAnidbRestrictedEntriesTest { + + private fun testAppConfig(dir: Directory) = object: Config by TestAppConfig { + override fun downloadControlStateDirectory(): Directory = dir + } + + @Test + fun `keeps added entries in the download control state directory`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + + // when + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // then + assertThat(tempDir.resolve(FILE_NAME).readLines().filterNot { it.startsWith("#") }) + .containsExactly("1202", "3531") + assertThat(DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)).all()) + .containsExactlyInAnyOrder(1202, 3531) + } + } + + @Test + fun `removes an entry`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // when + restrictedEntries.remove(3531) + + // then + assertThat(restrictedEntries.all()).containsExactly(1202) + } + } + + @Test + fun `removes entries which the title dump no longer names`() { + tempDirectory { + // given + val restrictedEntries = DefaultAnidbRestrictedEntries(appConfig = testAppConfig(tempDir)) + restrictedEntries.add(3531) + restrictedEntries.add(1202) + + // when + restrictedEntries.removeAllNotIn(setOf(3531, 1)) + + // then + assertThat(restrictedEntries.all()).containsExactly(3531) + } + } +} From 05d578df6e8ccfd19c96b233ed4cd648a571fe6c Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 19:33:21 +0000 Subject: [PATCH 36/50] Close challenged responses before asking again, AniDB maximum wait 3 s ClearanceHttpClient dropped a 403 response without closing it whenever it renewed the clearance or changed the connection, which OkHttp reported as a leaked connection. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/app/crawlers/anidb/AnidbCrawler.kt | 2 +- .../modb/app/network/ClearanceHttpClient.kt | 3 ++ .../app/network/ClearanceHttpClientTest.kt | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt index 224563d9..b449e4de 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/anidb/AnidbCrawler.kt @@ -296,7 +296,7 @@ class AnidbCrawler( private const val DEFAULT_MAX_CONNECTION_CHANGES = 5 private const val DEFAULT_ROUTE_FAILURE_WAIT_IN_SECONDS = 5 private const val DEFAULT_MIN_WAIT_IN_MILLISECONDS = 2000 - private const val DEFAULT_MAX_WAIT_IN_MILLISECONDS = 2500 + private const val DEFAULT_MAX_WAIT_IN_MILLISECONDS = 3000 private const val DEFAULT_PAGES_PER_CONNECTION = 0 /** diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt index e035937c..1bea7ace 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClient.kt @@ -67,6 +67,7 @@ class ClearanceHttpClient( renewedOnThisRoute = true if (clearance.refresh(url, route)) { + response.close() continue } @@ -80,6 +81,8 @@ class ClearanceHttpClient( return response } + // A refusal which isn't returned must be closed, otherwise its connection is never released. + response.close() changeConnection() renewedOnThisRoute = false } diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt index 920f0c08..e52a8cce 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/ClearanceHttpClientTest.kt @@ -133,6 +133,35 @@ internal class ClearanceHttpClientTest { } } + @Test + fun `closes every refusal it doesn't return`() { + runBlocking { + // given + val responses = mutableListOf() + val codes = mutableListOf(403, 403, 200) + val client = ClearanceHttpClient( + clearance = clearanceWith(solved), + httpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse { + return HttpResponse(codes.removeFirst(), "body").also { responses.add(it) } + } + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse = shouldNotBeInvoked() + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + }, + ) + + // when + val result = client.get(URI("https://anidb.net/anime/23").toURL()) + + // then + assertThat(result.code).isEqualTo(200) + assertThat(responses).hasSize(3) + assertThat(responses[0].isBodyInputStreamExhausted()).isTrue() + assertThat(responses[1].isBodyInputStreamExhausted()).isTrue() + assertThat(result.isBodyInputStreamExhausted()).isFalse() + } + } + @Test fun `renews the clearance once when the site refuses it`() { runBlocking { From a01c405b976a6cd9260e2f4cf6c35e21464e99cd Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 19:46:21 +0000 Subject: [PATCH 37/50] Log a stopped AniDB crawler after a refused connection without a stack trace Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index ceb3dfdd..cfaf8525 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -1,6 +1,7 @@ package io.github.manamiproject.modb.app import io.github.manamiproject.modb.anidb.AnidbBannedException +import io.github.manamiproject.modb.anidb.CrawlerDetectedException import io.github.manamiproject.modb.anidb.AnidbConfig import io.github.manamiproject.modb.anisearch.AnisearchConfig import io.github.manamiproject.modb.anisearch.AnisearchRelationsConfig @@ -213,6 +214,9 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: } catch (_: AnidbBannedException) { // Expected while a ban lasts, so a stack trace adds nothing. log.error("Crawler for [$hostname] stopped, because AniDB still answers with [banned] after all retries. Wait until the ban has been lifted or add [$hostname] to [modb.app.disabledCrawlers].") + } catch (_: CrawlerDetectedException) { + // AniDB refusing the connection is an answer, not a fault in the code, so a stack trace adds nothing. + log.error("Crawler for [$hostname] stopped, because AniDB refuses the connection (AntiLeech or 403). Try again later or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { log.error("Crawler for [$hostname] stopped: ${e.message}", e) } From ddc10a2ff873a6b7254035370082e3e292b05b6b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 19:57:23 +0000 Subject: [PATCH 38/50] Log every retry wait at info level with its reason and duration A retry waits up to minutes, but was only logged at debug and trace level, so a crawler waiting for its next attempt looked like a hung one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/core/httpclient/DefaultHttpClient.kt | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index 9018fa5d..c7be249f 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -199,7 +199,7 @@ public class DefaultHttpClient( okhttpClient = currentClient.newBuilder().build() } - wait(request, retryCase, attempt, retryAfter) + wait(request, retryCase, attempt, retryAfter, reasonOf(responseOrException)) } responseOrException = safelyExecute { @@ -245,18 +245,20 @@ public class DefaultHttpClient( } } + private fun reasonOf(responseOrException: Any): String = when (responseOrException) { + is HttpResponse -> "HTTP ${responseOrException.code}" + is Throwable -> listOfNotNull(responseOrException::class.simpleName, responseOrException.message).joinToString(": ") + else -> responseOrException.javaClass.simpleName + } + @KoverIgnore - private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int, retryAfter: Duration?) { + private suspend fun wait(request: Request, retryCase: RetryCase, attempt: Int, retryAfter: Duration?, reason: String) { if (!isTestContext) { - val duration = when (retryAfter) { - null -> retryCase.waitDuration(attempt) - else -> { - log.info { "Server asked to retry [${request.method} ${request.url}] after [$retryAfter]." } - retryAfter - } - } + val duration = retryAfter ?: retryCase.waitDuration(attempt) + val askedByServer = if (retryAfter != null) ", as the server asked" else EMPTY - log.trace { "Initiating waiting time of [$duration] for retry of [${request.method} ${request.url}]." } + // A wait can last minutes, so it is logged at info level: without it a waiting crawler looks like a hung one. + log.info { "Retrying [${request.method} ${request.url}] in [${duration.inWholeSeconds}s] (attempt $attempt/${retryBehavior.maxAttempts}) after [$reason]$askedByServer." } delay(duration) } } From 6c61327be664c2657c5f5299abaa32713107a75e Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 19:58:49 +0000 Subject: [PATCH 39/50] One-line log for a stopped crawler, clearer Livechart message when the year list is missing The stack trace of a stopped crawler moves to debug level. Livechart names the url and the page title it got instead of "Unable to extract headers." Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../main/kotlin/io/github/manamiproject/modb/app/App.kt | 4 +++- .../app/crawlers/livechart/LivechartNewestYearDetector.kt | 7 +++++-- .../crawlers/livechart/LivechartNewestYearDetectorTest.kt | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index cfaf8525..0b824fa4 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -218,7 +218,9 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: // AniDB refusing the connection is an answer, not a fault in the code, so a stack trace adds nothing. log.error("Crawler for [$hostname] stopped, because AniDB refuses the connection (AntiLeech or 403). Try again later or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { - log.error("Crawler for [$hostname] stopped: ${e.message}", e) + // The message says what happened. The stack trace is only of use when debugging. + log.error("Crawler for [$hostname] stopped: ${e.message ?: e::class.simpleName}") + log.debug("Stack trace of the stopped crawler for [$hostname]", e) } }) } diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt index 5a15fbaa..e76a2f31 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt @@ -33,17 +33,20 @@ class LivechartNewestYearDetector( wait() + val url = metaDataProviderConfig.buildDataDownloadLink().toURL() val response = httpClient.get( - url = metaDataProviderConfig.buildDataDownloadLink().toURL(), + url = url, headers = mapOf("host" to listOf("www.${metaDataProviderConfig.hostname()}")), ).checkedBody(this::class) val data = extractor.extract(response, mapOf( "headers" to "//ul[@class='grouped-list']/li//h5", + "title" to "//title/text()", )) if (data.notFound("headers")) { - throw IllegalStateException("Unable to extract headers.") + val title = data.stringOrDefault("title").trim().ifEmpty { "none" } + throw IllegalStateException("No list of years found on [$url], page title [$title]. Most likely a block or maintenance page, or the layout has changed.") } val headers = data.listNotNull("headers") diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt index 65d48df0..8053dd09 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt @@ -84,7 +84,7 @@ internal class LivechartNewestYearDetectorTest { } // then - assertThat(result).hasMessage("Unable to extract headers.") + assertThat(result).hasMessage("No list of years found on [http://localhost:8080/highest-id], page title [none]. Most likely a block or maintenance page, or the layout has changed.") } } } From 03d267e4d0fad9f7118156634774e156aba53295 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:14:28 +0000 Subject: [PATCH 40/50] Livechart: recognise a Cloudflare challenge, log it and retry it like any other failure The challenge page (403, or even 200) was taken as content and the crawler stopped on a missing year list. It now counts as a retry case, and the retry log names it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../core/httpclient/CloudflareChallenge.kt | 33 +++++++++ .../modb/core/httpclient/DefaultHttpClient.kt | 13 +++- .../httpclient/CloudflareChallengeKtTest.kt | 74 +++++++++++++++++++ .../crawlers/livechart/LivechartCrawler.kt | 3 +- .../livechart/LivechartNewestYearDetector.kt | 3 +- .../LivechartPaginationIdRangeSelector.kt | 3 +- 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt create mode 100644 core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt new file mode 100644 index 00000000..1eba3b86 --- /dev/null +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallenge.kt @@ -0,0 +1,33 @@ +package io.github.manamiproject.modb.core.httpclient + +/** + * Cloudflare answers a client it doesn't trust (yet) with an interstitial page instead of the content: the browser + * check "Just a moment..." or the block page "Attention Required!". Depending on the site this comes with 403, 503, + * 429 or even 200. + * + * Reads the body, which [HttpResponse] keeps in memory afterwards, so the caller can still read it. + * @since 1.0.0 + * @return `true` if the response is one of Cloudflare's interstitial pages rather than the requested content. + */ +public fun HttpResponse.isCloudflareChallenge(): Boolean { + if (code !in CHALLENGE_CODES) { + return false + } + + val body = bodyAsString() + return CHALLENGE_MARKERS.any { body.contains(it, ignoreCase = true) } +} + +/** + * Retries a request answered with a Cloudflare interstitial page, waiting like any other retry. + * @since 1.0.0 + */ +public val CloudflareChallengeRetryCase: HttpResponseRetryCase = HttpResponseRetryCase { it.isCloudflareChallenge() } + +private val CHALLENGE_CODES = setOf(200, 403, 429, 503) + +private val CHALLENGE_MARKERS = listOf( + "Just a moment...", + "Attention Required! | Cloudflare", + "window._cf_chl_opt", +) diff --git a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt index c7be249f..0e1c9819 100644 --- a/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt +++ b/core/src/main/kotlin/io/github/manamiproject/modb/core/httpclient/DefaultHttpClient.kt @@ -172,6 +172,7 @@ public class DefaultHttpClient( var retryCase: RetryCase = NoRetry var responseOrException: Any = HttpResponse(100, EMPTY) var retryAfter: Duration? = null + var reason = EMPTY do { if (retryCase !is NoRetry) { @@ -199,7 +200,7 @@ public class DefaultHttpClient( okhttpClient = currentClient.newBuilder().build() } - wait(request, retryCase, attempt, retryAfter, reasonOf(responseOrException)) + wait(request, retryCase, attempt, retryAfter, reason) } responseOrException = safelyExecute { @@ -216,6 +217,7 @@ public class DefaultHttpClient( log.debug { "[${request.method} ${request.url}] Requires retry." } retryAfter = null + reason = reasonOf(responseOrException) if (responseOrException is HttpResponse) { retryAfter = parseRetryAfter(responseOrException.headers) @@ -246,7 +248,14 @@ public class DefaultHttpClient( } private fun reasonOf(responseOrException: Any): String = when (responseOrException) { - is HttpResponse -> "HTTP ${responseOrException.code}" + // The body is only looked at if a retry case has read it already. Reading it here would download pages + // which are going to be discarded anyway. + is HttpResponse -> when { + responseOrException.isBodyInputStreamExhausted() && runCatching { responseOrException.isCloudflareChallenge() }.getOrDefault(false) -> { + "HTTP ${responseOrException.code}, Cloudflare challenge" + } + else -> "HTTP ${responseOrException.code}" + } is Throwable -> listOfNotNull(responseOrException::class.simpleName, responseOrException.message).joinToString(": ") else -> responseOrException.javaClass.simpleName } diff --git a/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt new file mode 100644 index 00000000..cdb71d7e --- /dev/null +++ b/core/src/test/kotlin/io/github/manamiproject/modb/core/httpclient/CloudflareChallengeKtTest.kt @@ -0,0 +1,74 @@ +package io.github.manamiproject.modb.core.httpclient + +import org.assertj.core.api.Assertions.assertThat +import kotlin.test.Test + +internal class CloudflareChallengeKtTest { + + private val challenge = """Just a moment...""" + private val content = """All Seasons | LiveChart.me
    """ + + @Test + fun `recognises the browser check`() { + // when + val result = HttpResponse(403, challenge).isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `recognises the browser check served with 200`() { + // when + val result = HttpResponse(200, challenge).isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `recognises the block page`() { + // when + val result = HttpResponse(403, "Attention Required! | Cloudflare").isCloudflareChallenge() + + // then + assertThat(result).isTrue() + } + + @Test + fun `regular content is no challenge`() { + // when + val result = HttpResponse(200, content).isCloudflareChallenge() + + // then + assertThat(result).isFalse() + } + + @Test + fun `an error page of another kind is no challenge`() { + // when + val result = HttpResponse(404, challenge).isCloudflareChallenge() + + // then + assertThat(result).isFalse() + } + + @Test + fun `the body can still be read afterwards`() { + // given + val response = HttpResponse(200, content) + + // when + response.isCloudflareChallenge() + + // then + assertThat(response.bodyAsString()).isEqualTo(content) + } + + @Test + fun `the retry case applies to a challenge only`() { + // then + assertThat(CloudflareChallengeRetryCase.retryIf(HttpResponse(403, challenge))).isTrue() + assertThat(CloudflareChallengeRetryCase.retryIf(HttpResponse(200, content))).isFalse() + } +} diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt index 65b38b2d..272f5ef6 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.config.AppConfig import io.github.manamiproject.modb.app.config.Config import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder @@ -56,7 +57,7 @@ class LivechartCrawler( private val paginationIdRangeSelector: PaginationIdRangeSelector = LivechartPaginationIdRangeSelector.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, private val downloader: Downloader = FaultTolerantDownloader( - downloader = LivechartDownloader(httpClient = SuspendableHttpClient()), + downloader = LivechartDownloader(httpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase)), hostname = metaDataProviderConfig.hostname(), ), ): Crawler { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt index e76a2f31..8e764750 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.crawlers.HighestIdDetector import io.github.manamiproject.modb.app.extensions.checkedBody import io.github.manamiproject.modb.app.network.SuspendableHttpClient @@ -24,7 +25,7 @@ import kotlin.time.toDuration */ class LivechartNewestYearDetector( private val metaDataProviderConfig: MetaDataProviderConfig = LivechartNewestYearDetectorConfig, - private val httpClient: HttpClient = SuspendableHttpClient(), + private val httpClient: HttpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase), private val extractor: DataExtractor = XmlDataExtractor, ): HighestIdDetector { diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt index 5f6516af..e53333e5 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.app.crawlers.livechart +import io.github.manamiproject.modb.core.httpclient.CloudflareChallengeRetryCase import io.github.manamiproject.modb.app.convfiles.AlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.convfiles.DefaultAlreadyDownloadedIdsFinder import io.github.manamiproject.modb.app.crawlers.PaginationIdRangeSelector @@ -35,7 +36,7 @@ import kotlin.time.toDuration */ class LivechartPaginationIdRangeSelector( private val metaDataProviderConfig: MetaDataProviderConfig = LivechartPaginationIdRangeSelectorConfig, - private val httpClient: HttpClient = SuspendableHttpClient(), + private val httpClient: HttpClient = SuspendableHttpClient().addRetryCases(CloudflareChallengeRetryCase), private val extractor: DataExtractor = XmlDataExtractor, private val downloadControlStateScheduler: DownloadControlStateScheduler = DefaultDownloadControlStateScheduler.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, From fc3eb76a266a2e297a84890b625677f8e93c67b3 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:21:15 +0000 Subject: [PATCH 41/50] Livechart: read the newest year from the season links of /charts The page no longer has the grouped list with year headings. The season links (/winter-2027, formerly /winter-2026/tv) are read instead, which work with either layout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../livechart/LivechartNewestYearDetector.kt | 19 +++++-- .../LivechartNewestYearDetectorTest.kt | 30 +++++++++++ .../charts-2026-10.html | 53 +++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt index 8e764750..871fb207 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetector.kt @@ -40,19 +40,23 @@ class LivechartNewestYearDetector( headers = mapOf("host" to listOf("www.${metaDataProviderConfig.hostname()}")), ).checkedBody(this::class) + // The season links are read instead of the headings, because they survive a change of the layout. val data = extractor.extract(response, mapOf( - "headers" to "//ul[@class='grouped-list']/li//h5", + "links" to "//a/@href", "title" to "//title/text()", )) - if (data.notFound("headers")) { + val years = when { + data.notFound("links") -> emptyList() + else -> data.listNotNull("links").mapNotNull { SEASON_LINK.find(it.trim())?.groupValues?.get(1)?.toInt() } + } + + if (years.isEmpty()) { val title = data.stringOrDefault("title").trim().ifEmpty { "none" } throw IllegalStateException("No list of years found on [$url], page title [$title]. Most likely a block or maintenance page, or the layout has changed.") } - val headers = data.listNotNull("headers") - - return headers[1].trim().toInt() + return years.max() } @KoverIgnore @@ -65,6 +69,11 @@ class LivechartNewestYearDetector( companion object { private val log by LoggerDelegate() + /** + * A season page, such as `/winter-2027` or `/winter-2026/tv`, either relative or as full link. + */ + private val SEASON_LINK = Regex("""^(?:https?://(?:www\.)?livechart\.me)?/(?:winter|spring|summer|fall)-(\d{4})(?:/.*)?$""") + /** * Singleton of [LivechartNewestYearDetector] * @since 1.0.0 diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt index 8053dd09..104df24d 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartNewestYearDetectorTest.kt @@ -49,6 +49,36 @@ internal class LivechartNewestYearDetectorTest { } } + @Test + fun `correctly extracts highest id from the layout of october 2026`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartNewestYearDetectorConfig { + override fun buildDataDownloadLink(id: String): URI = URI("http://localhost:8080/highest-id") + } + + val testHttpClient = object: HttpClient by TestHttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse { + return HttpResponse( + code = 200, + body = loadTestResource("crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html"), + ) + } + } + + val livechartHighestIdDetector = LivechartNewestYearDetector( + metaDataProviderConfig = testMetaDataProviderConfig, + httpClient = testHttpClient, + ) + + // when + val result = livechartHighestIdDetector.detectHighestId() + + // then + assertThat(result).isEqualTo(2027) + } + } + @Test fun `throws exception, because the element couldn't be found`() { runTest { diff --git a/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html b/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html new file mode 100644 index 00000000..0e750c64 --- /dev/null +++ b/lib/src/test/resources/crawler/livechart/LivechartNewestYearDetectorTest/charts-2026-10.html @@ -0,0 +1,53 @@ +All Seasons | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    All Seasons
    Season Unannounced
    TBA
    2027
    WinterSpring
    SummerFall
    2026 (Current)
    WinterSpring
    SummerFall
    2025
    WinterSpring
    SummerFall
    2024
    WinterSpring
    SummerFall
    2023
    WinterSpring
    SummerFall
    2022
    WinterSpring
    SummerFall
    2021
    WinterSpring
    SummerFall
    2020
    WinterSpring
    SummerFall
    2019
    WinterSpring
    SummerFall
    2018
    WinterSpring
    SummerFall
    2017
    WinterSpring
    SummerFall
    2016
    WinterSpring
    SummerFall
    2015
    WinterSpring
    SummerFall
    2014
    WinterSpring
    SummerFall
    2013
    WinterSpring
    SummerFall
    2012
    WinterSpring
    SummerFall
    2011
    WinterSpring
    SummerFall
    2010
    WinterSpring
    SummerFall
    2009
    WinterSpring
    SummerFall
    2008
    WinterSpring
    SummerFall
    2007
    WinterSpring
    SummerFall
    2006
    WinterSpring
    SummerFall
    2005
    WinterSpring
    SummerFall
    2004
    WinterSpring
    SummerFall
    2003
    WinterSpring
    SummerFall
    2002
    WinterSpring
    SummerFall
    2001
    WinterSpring
    SummerFall
    2000
    WinterSpring
    SummerFall
    1999
    WinterSpring
    SummerFall
    1998
    WinterSpring
    SummerFall
    1997
    WinterSpring
    SummerFall
    1996
    WinterSpring
    SummerFall
    1995
    WinterSpring
    SummerFall
    1994
    WinterSpring
    SummerFall
    1993
    WinterSpring
    SummerFall
    1992
    WinterSpring
    SummerFall
    1991
    WinterSpring
    SummerFall
    1990
    WinterSpring
    SummerFall
    1989
    WinterSpring
    SummerFall
    1988
    WinterSpring
    SummerFall
    1987
    WinterSpring
    SummerFall
    1986
    WinterSpring
    SummerFall
    1985
    WinterSpring
    SummerFall
    1984
    WinterSpring
    SummerFall
    1983
    WinterSpring
    SummerFall
    1982
    WinterSpring
    SummerFall
    1981
    WinterSpring
    SummerFall
    1980
    WinterSpring
    SummerFall
    1979
    WinterSpring
    SummerFall
    1978
    WinterSpring
    SummerFall
    1977
    WinterSpring
    SummerFall
    1976
    WinterSpring
    SummerFall
    1975
    WinterSpring
    SummerFall
    1974
    WinterSpring
    SummerFall
    1973
    WinterSpring
    SummerFall
    1972
    WinterSpring
    SummerFall
    1971
    WinterSpring
    SummerFall
    1970
    WinterSpring
    SummerFall
    1969
    WinterSpring
    SummerFall
    1968
    WinterSpring
    SummerFall
    1967
    WinterSpring
    SummerFall
    1966
    WinterSpring
    SummerFall
    1965
    WinterSpring
    SummerFall
    1964
    WinterSpring
    SummerFall
    1963
    WinterSpring
    SummerFall
    1962
    WinterSpring
    SummerFall
    1961
    WinterSpring
    SummerFall
    1960
    WinterSpring
    SummerFall
    1958
    WinterSpring
    SummerFall
    1957
    WinterSpring
    SummerFall
    1956
    WinterSpring
    SummerFall
    1946
    WinterSpring
    SummerFall
    1944
    WinterSpring
    SummerFall
    1943
    WinterSpring
    SummerFall
    1942
    WinterSpring
    SummerFall
    1936
    WinterSpring
    SummerFall
    1935
    WinterSpring
    SummerFall
    1934
    WinterSpring
    SummerFall
    1932
    WinterSpring
    SummerFall
    1931
    WinterSpring
    SummerFall
    1929
    WinterSpring
    SummerFall
    1917
    WinterSpring
    SummerFall
    1907
    WinterSpring
    SummerFall
    + + + + + + + + + + + \ No newline at end of file From a0b6936550bf7af5abf2c0284de49e6a237adcc9 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:25:45 +0000 Subject: [PATCH 42/50] Livechart: read the ids of the season pages from any card carrying one The cards no longer have the class "anime" since the redesign of october 2026. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../LivechartPaginationIdRangeSelector.kt | 3 +- .../LivechartPaginationIdRangeSelectorTest.kt | 38 +++++++++++++++++++ .../fall-2026-redesign.html | 12 ++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt index e53333e5..0067bd1e 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelector.kt @@ -63,7 +63,8 @@ class LivechartPaginationIdRangeSelector( } val data = extractor.extract(response.checkedBody(this::class), mapOf( - "animeIdList" to "//article[@class='anime']/@data-anime-id", + // Any card carrying an id: the class of the card changed with the redesign of october 2026. + "animeIdList" to "//article[@data-anime-id]/@data-anime-id", "emptyMessage" to "//section[@data-anime-card-list-target='emptyMessage']/text()", )) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt index 542771d8..d932b785 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartPaginationIdRangeSelectorTest.kt @@ -77,6 +77,44 @@ internal class LivechartPaginationIdRangeSelectorTest { } } + @Test + fun `correctly extracts page content of the layout of october 2026`() { + runTest { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartPaginationIdRangeSelectorConfig { + override fun isTestContext(): Boolean = true + } + + val testHttpClient = object: HttpClient by TestHttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = HttpResponse( + code = 200, + body = loadTestResource("crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html") + ) + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val livechartPaginationIdRangeSelector = LivechartPaginationIdRangeSelector( + metaDataProviderConfig = testMetaDataProviderConfig, + httpClient = testHttpClient, + downloadControlStateScheduler = testDownloadControlStateScheduler, + alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, + ) + + // when + val result = livechartPaginationIdRangeSelector.idDownloadList("fall-2026") + + // then + assertThat(result).containsExactlyInAnyOrder("13226", "13232", "11748") + } + } + @Test fun `removes anime not scheduled for the current week`() { runTest { diff --git a/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html b/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html new file mode 100644 index 00000000..ed76f088 --- /dev/null +++ b/lib/src/test/resources/crawler/livechart/LivechartPaginationIdRangeSelectorTest/fall-2026-redesign.html @@ -0,0 +1,12 @@ + + + +Fall 2026 Anime Chart - All | LiveChart.me + + +
    +
    +
    +
    + + From 0e45e1472940538c19ac6f3ad624f581310706fa Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:37:31 +0000 Subject: [PATCH 43/50] Livechart: recheck the current and upcoming seasons and tba on every run Once all season pages had been crawled in a week, anime added to livechart in the meantime were only found the next week. The pages from the current season up to the last season of the newest year, and tba, are now requested on every run. They aren't memorized as last page and only new ids are downloaded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../crawlers/livechart/LivechartCrawler.kt | 39 ++++++++ .../livechart/LivechartCrawlerTest.kt | 93 +++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt index 272f5ef6..bd791484 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawler.kt @@ -28,6 +28,8 @@ import io.github.manamiproject.modb.core.random import io.github.manamiproject.modb.livechart.LivechartConfig import io.github.manamiproject.modb.livechart.LivechartDownloader import kotlinx.coroutines.delay +import java.time.Clock +import java.time.LocalDate import kotlin.time.DurationUnit.MILLISECONDS import kotlin.time.toDuration @@ -54,6 +56,7 @@ class LivechartCrawler( private val deadEntriesAccessor: DeadEntriesAccessor = DefaultDeadEntriesAccessor.instance, private val lastPageMemorizer: LastPageMemorizer = StringBasedLastPageMemorizer(metaDataProviderConfig = metaDataProviderConfig), private val newestYearDetector: HighestIdDetector = LivechartNewestYearDetector.instance, + private val clock: () -> Clock = { appConfig.clock() }, private val paginationIdRangeSelector: PaginationIdRangeSelector = LivechartPaginationIdRangeSelector.instance, private val alreadyDownloadedIdsFinder: AlreadyDownloadedIdsFinder = DefaultAlreadyDownloadedIdsFinder.instance, private val downloader: Downloader = FaultTolerantDownloader( @@ -102,13 +105,31 @@ class LivechartCrawler( wait() + val handledInThisRun = hashSetOf() + pages.forEach { page -> val currentList = paginationIdRangeSelector.idDownloadList(page) - entriesNotScheduledForCurrentWeek - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) wait() startDownload(currentList) + handledInThisRun.addAll(currentList) lastPageMemorizer.memorizeLastPage(page) } + // New anime appear in the current and the upcoming seasons, so these are looked at on every run, even once + // every page has been crawled this week. They aren't memorized, so they don't move where the crawl resumes. + val recheck = recheckPages(newestYear) - pages.toSet() + + if (recheck.isNotEmpty()) { + log.info { "[${metaDataProviderConfig.hostname()}] Rechecking [${recheck.size}] pages for new anime: $recheck" } + } + + recheck.forEach { page -> + val currentList = paginationIdRangeSelector.idDownloadList(page) - entriesNotScheduledForCurrentWeek - alreadyDownloadedIdsFinder.alreadyDownloadedIds(metaDataProviderConfig) - handledInThisRun + wait() + startDownload(currentList) + handledInThisRun.addAll(currentList) + } + log.info { "Finished downloading [${metaDataProviderConfig.hostname()}] entries using pagination." } } @@ -124,6 +145,19 @@ class LivechartCrawler( .sorted() } + /** + * Pages from the current season up to the last season of the newest year, followed by `tba`. + */ + private fun recheckPages(newestYear: Year): List { + val today = LocalDate.now(clock()) + val currentSeason = (today.monthValue - 1) / 3 + + return (today.year..newestYear).flatMap { year -> + SEASONS_IN_ORDER.filterIndexed { index, _ -> year > today.year || index >= currentSeason } + .map { season -> "$season-$year" } + } + "tba" + } + private suspend fun startDownload(idDownloadList: List) = repeat(idDownloadList.size) { index -> val animeId = idDownloadList[index] val file = appConfig.workingDir(metaDataProviderConfig).resolve("$animeId.${metaDataProviderConfig.fileSuffix()}") @@ -151,6 +185,11 @@ class LivechartCrawler( companion object { private val log by LoggerDelegate() + /** + * Seasons in the order of the year: January to March is winter, October to December is fall. + */ + private val SEASONS_IN_ORDER = listOf("winter", "spring", "summer", "fall") + /** * Singleton of [LivechartCrawler] * @since 1.0.0 diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt index 127b9a16..1b436600 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt @@ -21,10 +21,16 @@ import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatNoException import org.junit.jupiter.api.Nested +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset.UTC import kotlin.test.Test internal class LivechartCrawlerTest { + // Lies after every newest year of these tests, so only "tba" is rechecked. + private val farFuture = Clock.fixed(Instant.parse("3000-01-01T00:00:00Z"), UTC) + @Nested inner class StartTests { @@ -69,6 +75,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = TestDownloader, @@ -138,6 +145,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -221,6 +229,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -294,6 +303,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -370,6 +380,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -444,6 +455,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -516,6 +528,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -529,6 +542,82 @@ internal class LivechartCrawlerTest { } } + @Test + fun `rechecks the pages from the current season on and tba on every run without memorizing them`() { + tempDirectory { + // given + val testMetaDataProviderConfig = object: MetaDataProviderConfig by LivechartConfig { + override fun isTestContext(): Boolean = true + } + + val testAppConfig = object: Config by TestAppConfig { + override fun workingDir(metaDataProviderConfig: MetaDataProviderConfig): Directory = tempDir + } + + val testDownloadControlStateScheduler = object: DownloadControlStateScheduler by TestDownloadControlStateScheduler { + override suspend fun findEntriesScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + override suspend fun findEntriesNotScheduledForCurrentWeek(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val memorized = mutableListOf() + val testLastPageMemorizer = object: LastPageMemorizer by TestLastPageMemorizerString { + override suspend fun retrieveLastPage(): String = "winter-2027" + override suspend fun memorizeLastPage(page: String) { + memorized.add(page) + } + } + + val testNewestYearDetector = object: HighestIdDetector by TestHighestIdDetector { + override suspend fun detectHighestId(): Int = 2027 + } + + val requested = mutableListOf() + val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { + override suspend fun idDownloadList(page: String): List { + requested.add(page) + return when (page) { + "fall-2026" -> listOf("new-1", "new-2") + "tba" -> listOf("new-2", "new-3") + else -> emptyList() + } + } + } + + val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { + override suspend fun alreadyDownloadedIds(metaDataProviderConfig: MetaDataProviderConfig): Set = emptySet() + } + + val downloadedEntries = mutableListOf() + val testDownloader = object: Downloader by TestDownloader { + override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String { + downloadedEntries.add(id) + return id + } + } + + val crawler = LivechartCrawler( + appConfig = testAppConfig, + metaDataProviderConfig = testMetaDataProviderConfig, + downloadControlStateScheduler = testDownloadControlStateScheduler, + deadEntriesAccessor = TestDeadEntriesAccessor, + lastPageMemorizer = testLastPageMemorizer, + newestYearDetector = testNewestYearDetector, + clock = { Clock.fixed(Instant.parse("2026-10-30T12:00:00Z"), UTC) }, + paginationIdRangeSelector = testPaginationIdRangeSelector, + alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, + downloader = testDownloader, + ) + + // when + crawler.start() + + // then + assertThat(requested).containsExactly("fall-2026", "winter-2027", "spring-2027", "summer-2027", "fall-2027", "tba") + assertThat(downloadedEntries).containsExactly("new-1", "new-2", "new-3") + assertThat(memorized).isEmpty() + } + } + @Test fun `resumes on last memorized page`() { tempDirectory { @@ -581,6 +670,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -651,6 +741,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = testDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -718,6 +809,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = testDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = testDownloader, @@ -776,6 +868,7 @@ internal class LivechartCrawlerTest { deadEntriesAccessor = TestDeadEntriesAccessor, lastPageMemorizer = testLastPageMemorizer, newestYearDetector = testNewestYearDetector, + clock = { farFuture }, paginationIdRangeSelector = testPaginationIdRangeSelector, alreadyDownloadedIdsFinder = testAlreadyDownloadedIdsFinder, downloader = TestDownloader, From 0cd6ab7988fc06fd7b53ac21b6eb41ee9fbe579c Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:40:47 +0000 Subject: [PATCH 44/50] Livechart tests: keep the recheck of tba out of the pagination tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../livechart/LivechartCrawlerTest.kt | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt index 1b436600..dcd2eb93 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/crawlers/livechart/LivechartCrawlerTest.kt @@ -279,9 +279,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { @@ -353,9 +354,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { @@ -431,9 +433,10 @@ internal class LivechartCrawlerTest { } val testPaginationIdRangeSelector = object: PaginationIdRangeSelector by TestPaginationIdRangeSelectorString { - override suspend fun idDownloadList(page: String): List = listOf( - "paginated-entry-$page", - ) + override suspend fun idDownloadList(page: String): List = when (page) { + "tba" -> emptyList() // rechecked on every run, which has a test of its own + else -> listOf("paginated-entry-$page") + } } val testAlreadyDownloadedIdsFinder = object: AlreadyDownloadedIdsFinder by TestAlreadyDownloadedIdsFinder { From eb85e11878e16cf086a0a5e92af0d9490306151f Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:50:28 +0000 Subject: [PATCH 45/50] Livechart: convert anime pages in the layout of october 2026 The values no longer follow their label as text, but sit in the next element. Type, status, duration, season, year, episodes, the next episode, related anime and tags are read from the new layout first, with the earlier selectors as fallback. Blank studio names from the navigation are dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/livechart/LivechartAnimeConverter.kt | 68 ++++++++++----- .../livechart/LivechartAnimeConverterTest.kt | 80 +++++++++++++++++ .../layout_2026_10/11343.html | 86 +++++++++++++++++++ .../layout_2026_10/11428.html | 86 +++++++++++++++++++ .../layout_2026_10/12345.html | 86 +++++++++++++++++++ 5 files changed, 386 insertions(+), 20 deletions(-) create mode 100644 livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html create mode 100644 livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html create mode 100644 livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html diff --git a/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt b/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt index d4a98e23..6759e589 100644 --- a/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt +++ b/livechart/src/main/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverter.kt @@ -54,6 +54,18 @@ public class LivechartAnimeConverter( "sourceDiv" to "//div[@data-anime-details-id]/@data-anime-details-id", "sourceMeta" to "//meta[@property='og:url']/@content", "studios" to "//a[contains(@href, 'studios')]/text()", + // Layout since the redesign of october 2026: the value sits in the element following its label. The keys + // above stay as fallback for pages in the earlier layout. + "typeValue" to "//div[text()='Format']/following-sibling::div[1]/text()", + "statusValue" to "//div[text()='Status']/following-sibling::*[1]/text()", + "durationValue" to "//div[text()='Run time']/following-sibling::div[1]/text()", + "seasonValue" to "//div[text()='Season']/following-sibling::a[1]/text()", + "yearValue" to "//div[text()='Premiere']/following-sibling::*[1]/text()", + "episodesDetails" to "//div[@data-anime-details-target='viewingProgression']/@data-viewing-progression-anime-id", + "episodesTotal" to "//div[@data-anime-details-target='viewingProgression']/@data-viewing-progression-total-count-value", + "episodesNext" to "//div[a[contains(@href,'/schedules/')]]//strong[starts-with(text(),'EP')]/text()", + "relatedAnimeCards" to "//article[@data-related-anime-target='anime']//a/@href", + "tagChips" to "//div[text()='Tags']/following-sibling::div[1]//a/text()", )) val jsonld = data.listNotNull("jsonld").first() @@ -104,8 +116,14 @@ public class LivechartAnimeConverter( var episodes = jsonData.intOrDefault("numberOfEpisodes") var isTableValueUnknown = false - // regular value in data table - if (episodes == 0) { + // number of episodes of the details card, layout since october 2026 + if (episodes == 0 && !data.notFound("episodesDetails")) { + episodes = data.firstText("episodesTotal").toIntOrNull() ?: 0 + isTableValueUnknown = episodes == 0 + } + + // regular value in data table, earlier layout + if (episodes == 0 && data.notFound("episodesDetails")) { val episodesValue = data.stringOrDefault("episodesDiv", "?") .trim() .split('/') @@ -120,7 +138,7 @@ public class LivechartAnimeConverter( // current episode from table if the anime is ongoing if (episodes == 0) { - episodes = data.stringOrDefault("episodesCountdown") + episodes = data.firstText("episodesNext", "episodesCountdown") .remove("EP") .trim() .toIntOrNull() @@ -138,7 +156,7 @@ public class LivechartAnimeConverter( } private fun extractType(data: ExtractionResult): AnimeType { - val value = data.string("type") + val value = data.firstText("typeValue", "type") return when(value.trim().lowercase()) { "movie" -> MOVIE @@ -176,7 +194,7 @@ public class LivechartAnimeConverter( } private fun extractStatus(data: ExtractionResult): AnimeStatus { - val statusString = data.stringOrDefault("status") + val statusString = data.firstText("statusValue", "status") return when (statusString.trim().lowercase()){ "not yet released" -> UPCOMING @@ -187,7 +205,7 @@ public class LivechartAnimeConverter( } private fun extractDuration(data: ExtractionResult): Duration { - val durationString = data.stringOrDefault("duration").trim() + val durationString = data.firstText("durationValue", "duration").trim() val seconds = """(\d+ ?[aA-zZ]+)+""".toRegex() .findAll(durationString) @@ -211,10 +229,11 @@ public class LivechartAnimeConverter( } private fun extractAnimeSeason(data: ExtractionResult): AnimeSeason { - val splitSeasonString = if (data.notFound("season")) { + val seasonText = data.firstText("seasonValue", "season") + val splitSeasonString = if (seasonText.isBlank()) { listOf(EMPTY) } else { - data.listNotNull("season").first().remove("Season ").split(' ') + seasonText.remove("Season ").split(' ') } val seasonString = splitSeasonString.first().trim().lowercase() @@ -229,7 +248,7 @@ public class LivechartAnimeConverter( val year = if (splitSeasonString.size == 2) { YEAR_REGEX.find(splitSeasonString[1])?.value?.trim()?.ifBlank { "0" }?.toIntOrNull() ?: 0 } else { - YEAR_REGEX.find(data.stringOrDefault("year"))?.value?.trim()?.toIntOrNull() ?: 0 + YEAR_REGEX.find(data.firstText("yearValue", "year"))?.value?.trim()?.toIntOrNull() ?: 0 } return AnimeSeason( @@ -261,14 +280,12 @@ public class LivechartAnimeConverter( } private fun extractRelatedAnime(data: ExtractionResult): HashSet { - return if (data.notFound("relatedAnime")) { - hashSetOf() - } else { - data.listNotNull("relatedAnime") - .map { it.remove("/anime/") } - .map { metaDataProviderConfig.buildAnimeLink(it) } - .toHashSet() - } + val key = listOf("relatedAnimeCards", "relatedAnime").firstOrNull { !data.notFound(it) } ?: return hashSetOf() + + return data.listNotNull(key) + .mapNotNull { ANIME_LINK.find(it.trim())?.groupValues?.get(1) } + .map { metaDataProviderConfig.buildAnimeLink(it) } + .toHashSet() } private fun extractTags(jsonData: ExtractionResult, data: ExtractionResult): HashSet { @@ -280,9 +297,10 @@ public class LivechartAnimeConverter( .toHashSet() } - if (!data.notFound("tags")) { - data.listNotNull("tags") + listOf("tagChips", "tags").firstOrNull { !data.notFound(it) }?.let { key -> + data.listNotNull(key) .map { it.trim().lowercase() } + .filter { it.isNotBlank() } .forEach { tags.add(it) } } @@ -315,7 +333,7 @@ public class LivechartAnimeConverter( if (!data.notFound("studios")) { data.listNotNull("studios") .map { it.normalize().lowercase() } - .filterNot { it == "studios" } + .filterNot { it == "studios" || it.isBlank() } .forEach { studios.add(it) } } @@ -326,6 +344,16 @@ public class LivechartAnimeConverter( private const val LARGE_PICTURE_INDICATOR = "large.jpg" private const val SMALL_PICTURE_INDICATOR = "small.jpg" private val YEAR_REGEX = """\d{4}""".toRegex() + private val ANIME_LINK = """^(?:https?://(?:www\.)?livechart\.me)?/anime/(\d+)/?$""".toRegex() + + /** + * Text of the first key which has been found and isn't blank. A key can hold one value or a list of values. + */ + private fun ExtractionResult.firstText(vararg keys: String): String = keys.asSequence() + .filterNot { notFound(it) } + .map { listNotNull(it) { value -> value }.joinToString(" ").trim() } + .firstOrNull { it.isNotBlank() } + ?: EMPTY /** * Singleton of [LivechartAnimeConverter] diff --git a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt index b7527f4e..8585012f 100644 --- a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt +++ b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/LivechartAnimeConverterTest.kt @@ -1385,6 +1385,86 @@ internal class LivechartAnimeConverterTest { } } + private fun converter() = LivechartAnimeConverter( + metaDataProviderConfig = object : MetaDataProviderConfig by TestMetaDataProviderConfig { + override fun hostname(): Hostname = LivechartConfig.hostname() + override fun buildAnimeLink(id: AnimeId): URI = LivechartConfig.buildAnimeLink(id) + override fun fileSuffix(): FileSuffix = LivechartConfig.fileSuffix() + }, + ) + + @Nested + inner class LayoutOfOctober2026Tests { + + @Test + fun `finished anime`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/12345.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.type).isEqualTo(TV) + assertThat(result.status).isEqualTo(FINISHED) + assertThat(result.episodes).isEqualTo(12) + assertThat(result.duration).isEqualTo(Duration(24, MINUTES)) + assertThat(result.animeSeason.season).isEqualTo(SUMMER) + assertThat(result.animeSeason.year).isEqualTo(2026) + assertThat(result.tags).contains("action", "adventure", "fantasy") + assertThat(result.studios).contains("maho film") + assertThat(result.studios).doesNotContain("") + assertThat(result.relatedAnime).isEmpty() + } + } + + @Test + fun `airing sequel`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/11343.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.type).isEqualTo(TV) + assertThat(result.status).isEqualTo(ONGOING) + assertThat(result.episodes).isEqualTo(13) + assertThat(result.animeSeason.season).isEqualTo(FALL) + assertThat(result.animeSeason.year).isEqualTo(2026) + assertThat(result.relatedAnime).containsExactly( + URI("https://livechart.me/anime/10355"), + ) + } + } + + @Test + fun `airing anime without a known number of episodes takes the episode before the next one`() { + runTest { + // given + val testFile = loadTestResource("LivechartAnimeConverterTest/layout_2026_10/11428.html") + + // when + val result = converter().convert(testFile) + + // then + assertThat(result.status).isEqualTo(ONGOING) + assertThat(result.episodes).isEqualTo(1751) + assertThat(result.animeSeason.season).isEqualTo(FALL) + assertThat(result.animeSeason.year).isEqualTo(1988) + assertThat(result.relatedAnime).containsExactlyInAnyOrder( + URI("https://livechart.me/anime/11443"), + URI("https://livechart.me/anime/11430"), + URI("https://livechart.me/anime/11397"), + URI("https://livechart.me/anime/7249"), + URI("https://livechart.me/anime/11405"), + ) + } + } + } + @Nested inner class CompanionObjectTests { diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html new file mode 100644 index 00000000..84dbfa0d --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11343.html @@ -0,0 +1,86 @@ + +The Detective is Already Dead Season 2 | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Tantei wa Mou, Shindeiru. Season2
    English title
    The Detective is Already Dead Season 2
    Native title
    探偵はもう、死んでいる。Season2
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Fall 2026)
    7.9/10
    181 Ratings
    –/10
    You
    EP5 · TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Format
    TV
    Episodes
    ?/13
    Run time
    24m

    After thwarting a hijacking with beautiful, silver-haired detective Siesta, Kimihiko leads a life of globe-trotting and crime-solving—until Siesta dies. A year later, a new case will open old wounds as it threads murder, conspiracy, and his dead partner!

    [Source: Crunchyroll]

    Studio
    ENGI
    Tags
    ComedyDramaMysteryRomanceSequel
    Videos
    Play video: PV 3
    :51
    JAEN
    PV 3
    Play video: PV 2
    :24
    JAEN
    PV 2
    Streams
    Crunchyroll
    Sub & Dub (English, French, German, Hindi, Portuguese (Brazil), Spanish (Latin America))
    Franchise: The Detective is Already Dead
    The Detective is Already Dead
    • TV
    • 2021
    • 7.32
    ?/12
    Show more
    WebsiteOfficial X
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearchKitsu
    + + + + + + + + + + + + \ No newline at end of file diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html new file mode 100644 index 00000000..278ff639 --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/11428.html @@ -0,0 +1,86 @@ + +Sore Ike! Anpanman | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Sore Ike! Anpanman
    Native title
    それいけ！アンパンマン
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Fall 1988)
    EP1752 · TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Format
    TV
    Episodes
    ?/–
    Run time
    24m

    One night, a Star of Life falls down the chimney of a bakery nestled deep in the forest, causing the dough in the oven to come to life. The dough becomes Anpanman, a superhero made of anpan (a sweet roll with bean jam filling). Together with his friends, Anpanman fights his rival Baikinman and helps the malnourished.

    [Source: Anime News Network]

    Studio
    TMS Entertainment
    Tags
    ComedyFantasyKids
    Franchise: Anpanman
    Go! Anpanman: The Shining Star's Tear
    • Movie
    • 1989
    ?/1
    Sore Ike! Anpanman: Baikinman no Gyakushuu
    • Movie
    • 1990
    ?/1
    Sore Ike! Anpanman: Omusubiman
    • Movie
    • 1990
    ?/1
    Sore Ike! Anpanman: Minami no Umi wo Sukue!
    • Special
    • 1990
    ?/1
    Sore Ike! Anpanman: Dokin-chan no Dokidoki Calendar
    • Movie
    • 1991
    ?/1
    Show more
    Website
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearchKitsu
    + + + + + + + + + + + + \ No newline at end of file diff --git a/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html new file mode 100644 index 00000000..5492d4c2 --- /dev/null +++ b/livechart/src/test/resources/LivechartAnimeConverterTest/layout_2026_10/12345.html @@ -0,0 +1,86 @@ + +The Insipid Prince's Furtive Grab for The Throne | LiveChart.me
    Current Season
    Explore
    Anime
    Franchises
    Studios
    Tags
    Schedule
    Seasons
    Winter
    Spring
    Summer
    Fall
    TBA
    All
    HeadlinesVideos
    AboutSettings
    Log in
    Account
    Settings
    Library import
    Database submissions
    What's new
    Light
    Dark
    Device
    Log out
    Titles
    Native title (romanized)
    Saikyou Degarashi Ouji no Anyaku Teii Arasoi: Munou wo Enjiru SS Rank Ouji wa Koui Keishousen wo Kage kara Shihai suru
    English title
    The Insipid Prince's Furtive Grab for The Throne
    Native title
    最強出涸らし皇子の暗躍帝位争い 無能を演じるSSランク皇子は皇位継承戦を影から支配する
    Info
    Schedules
    Videos
    Streams
    Visuals
    Headlines
    (Summer 2026)
    7.5/10
    832 Ratings
    –/10
    You
    TV (JP)
    ?
    ?
    ?
    ?
    Days
    Hrs
    Mins
    Sec
    Released
    Format
    TV
    Episodes
    ?/12
    Run time
    24m

    To the empire, Prince Arnold is one big joke, a completely inept prince outshone by his more talented twin, Leonard. But in secret, Arnold operates as Silver, one of only five magic-wielding SS-rank adventurers on the continent. As the bloody battle for the throne intensifies after the crown prince’s death, Arnold pulls the strings from the shadows to crown his twin emperor to his own advantage.

    [Source: Crunchyroll]

    Studio
    MAHO FILM
    Tags
    ActionAdventureFantasy
    Videos
    Play video: PV 2
    :51
    JANone
    PV 2
    Play video: PV 1
    :10
    JANone
    PV 1
    Streams
    Crunchyroll
    Sub & Dub (Thai)
    WebsiteOfficial X
    External resources
    AniListMyAnimeListAniDBAnime-PlanetaniSearch
    + + + + + + + + + + + + \ No newline at end of file From cb82ed72283b1a0ee83108bd82b57bfddd516099 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 20:56:55 +0000 Subject: [PATCH 46/50] Livechart: add the pages of the new layout to the test resource update list Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../manamiproject/modb/livechart/UpdateTestResources.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt index 346e6acf..75ea064f 100644 --- a/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt +++ b/livechart/src/test/kotlin/io/github/manamiproject/modb/livechart/UpdateTestResources.kt @@ -42,6 +42,10 @@ private val files = mapOf( "LivechartAnimeConverterTest/episodes/number_of_episodes_unknown_but_currently_running.html" to "319", "LivechartAnimeConverterTest/episodes/unknown.html" to "11873", + "LivechartAnimeConverterTest/layout_2026_10/11343.html" to "11343", + "LivechartAnimeConverterTest/layout_2026_10/11428.html" to "11428", + "LivechartAnimeConverterTest/layout_2026_10/12345.html" to "12345", + "LivechartAnimeConverterTest/picture_and_thumbnail/neither_picture_nor_thumbnail.html" to "2301", "LivechartAnimeConverterTest/picture_and_thumbnail/picture_and_thumbnail_available.html" to "3437", From c32d29c9517818484cac43e814a8c4cfc81a2484 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 21:02:38 +0000 Subject: [PATCH 47/50] Log the stack trace of a stopped crawler at trace level only Debug is the default level of the log, so the stack trace still showed up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt index 0b824fa4..78382a89 100644 --- a/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt +++ b/app/src/main/kotlin/io/github/manamiproject/modb/app/App.kt @@ -218,9 +218,10 @@ private fun CoroutineScope.crawl(hostname: String, disabled: Set, jobs: // AniDB refusing the connection is an answer, not a fault in the code, so a stack trace adds nothing. log.error("Crawler for [$hostname] stopped, because AniDB refuses the connection (AntiLeech or 403). Try again later or add [$hostname] to [modb.app.disabledCrawlers].") } catch (e: Throwable) { - // The message says what happened. The stack trace is only of use when debugging. + // The message says what happened. The stack trace is only of use when debugging, so it is logged at trace + // level, which the regular log levels (down to debug) leave out. log.error("Crawler for [$hostname] stopped: ${e.message ?: e::class.simpleName}") - log.debug("Stack trace of the stopped crawler for [$hostname]", e) + log.trace("Stack trace of the stopped crawler for [$hostname]", e) } }) } From 6076a305fbe79d8978351534abdd27c9127b2334 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 22:49:17 +0000 Subject: [PATCH 48/50] AniDB API: log the refusal without claiming the id becomes dead The crawler decides between dead and pending and logs that itself. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt index d26683aa..ada20846 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiDownloader.kt @@ -103,7 +103,8 @@ public class AnidbApiDownloader( // Anime not found, so a check on the code alone lets that through and the // apology page ends up stored and converted as if it were an anime. if (code in DEAD_ENTRY_CODES || message.lowercase() in DEAD_ENTRY_MESSAGES) { - log.info { "Adding [anidbId=$id] to dead entries list, because the API responded with [$message]." } + // Whether the id is dead or still pending is decided by the caller, which logs the outcome. + log.info { "The API responded with [$message] for [anidbId=$id]." } onDeadEntry.invoke(id) return EMPTY_RESPONSE } From 4aa1846d1fcd0d12f9b627c106899bcf12faac6b Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 22:51:10 +0000 Subject: [PATCH 49/50] Shorter config template Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- config.example.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config.example.toml b/config.example.toml index 13e1f27b..a7e4e502 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,10 +1,7 @@ -# Copy to config.toml and fill in the values. All directories must exist. - [modb.app] downloadsDirectory = "" outputDirectory = "" downloadControlStateDirectory = "" -# Crawlers to skip. disabledCrawlers = ["anidb.net"] [modb.app.challenge] @@ -26,4 +23,3 @@ client = "" clientver = "" # Retries while AniDB answers "banned", before the AniDB crawler stops. banRetries = 5 - From 6264335905577eeac07880579ddb2f319f58fcd1 Mon Sep 17 00:00:00 2001 From: Christian Gambale Date: Fri, 30 Oct 2026 23:06:33 +0000 Subject: [PATCH 50/50] Name the challenge solver when it cannot be reached OkHttp's message only names localhost:8191, which reads as if the site itself were unreachable. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVzhaDVosMJJcWUJS3p5Sj --- .../modb/app/network/CloudflareClearance.kt | 10 ++++++- .../app/network/CloudflareClearanceTest.kt | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt index 3821f538..27ae6b5b 100644 --- a/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt +++ b/lib/src/main/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearance.kt @@ -11,6 +11,7 @@ import io.github.manamiproject.modb.core.json.Json import io.github.manamiproject.modb.core.logging.LoggerDelegate import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.net.ConnectException import java.net.URI import java.net.URL import java.nio.file.Files @@ -125,7 +126,14 @@ class CloudflareClearance( body = """{"cmd":"request.get","url":"$url","maxTimeout":$SOLVER_TIMEOUT,"disableMedia":true$proxy}""", ) - val response = httpClient.post(URI(solverUrl).toURL(), body).bodyAsString() + val response = try { + httpClient.post(URI(solverUrl).toURL(), body).bodyAsString() + } catch (e: ConnectException) { + // OkHttp's message only names an address, which reads as if the site itself were unreachable. + // It stays a ConnectException, so the callers' handling doesn't change. + val address = e.message?.substringAfter("Failed to connect to ", missingDelimiterValue = "")?.ifBlank { null } ?: solverUrl + throw ConnectException("Failed to connect to the challenge solver [$address]").apply { initCause(e) } + } val parsed = Json.parseJson(response) if (parsed?.status != "ok" || parsed.solution == null) { diff --git a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt index cb780047..1d7369a2 100644 --- a/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt +++ b/lib/src/test/kotlin/io/github/manamiproject/modb/app/network/CloudflareClearanceTest.kt @@ -11,6 +11,8 @@ import io.github.manamiproject.modb.test.tempDirectory import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.assertThrows +import java.net.ConnectException import java.net.URI import java.net.URL import java.nio.file.Path @@ -48,6 +50,33 @@ internal class CloudflareClearanceTest { } } + @Test + fun `names the challenge solver if it cannot be reached`() { + runBlocking { + // given + val clearance = CloudflareClearance( + configRegistry = registry(file = null), + httpClient = object: HttpClient { + override suspend fun get(url: URL, headers: Map>): HttpResponse = shouldNotBeInvoked() + override suspend fun post(url: URL, requestBody: RequestBody, headers: Map>): HttpResponse { + throw ConnectException("Failed to connect to localhost/127.0.0.1:8191") + } + override fun addRetryCases(vararg retryCases: RetryCase): HttpClient = this + }, + hostname = "anidb.net", + ) + + // when + val result = assertThrows { + runBlocking { clearance.refresh(URI("https://anidb.net/anime/23").toURL()) } + } + + // then + assertThat(result).hasMessage("Failed to connect to the challenge solver [localhost/127.0.0.1:8191]") + assertThat(result.cause).isInstanceOf(ConnectException::class.java) + } + } + @Test fun `holds a clearance per route`() { runBlocking {