diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverter.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverter.kt index 89ec3935..f814dbea 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverter.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverter.kt @@ -9,6 +9,7 @@ import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank import io.github.manamiproject.modb.core.extensions.remove import io.github.manamiproject.modb.core.extractor.DataExtractor import io.github.manamiproject.modb.core.extractor.ExtractionResult +import io.github.manamiproject.modb.core.extractor.extractLabelledLinks import io.github.manamiproject.modb.core.extractor.XmlDataExtractor import io.github.manamiproject.modb.core.anime.* import io.github.manamiproject.modb.core.anime.AnimeMedia.NO_PICTURE @@ -53,7 +54,7 @@ public class AnidbAnimeConverter( "season" to "//tr[contains(@class, 'season')]//td[contains(@class, 'value')]/text()", "startDate" to "//tr[contains(@class, 'year')]//td[contains(@class, 'value')]//span[contains(@itemprop, 'startDate')]/@content", "datePublished" to "//tr[contains(@class, 'year')]//td[contains(@class, 'value')]//span[contains(@itemprop, 'datePublished')]/@content", - "relatedAnime" to "//div[contains(@class, 'directly_related')]//a/@href", + "relatedAnime" to "//div[contains(@class, 'directly_related')]//div[contains(@class, 'stripe')]/html()", "alternateNames" to "//label[contains(@itemprop, 'alternateName')]/text()", "synonymsList" to "//div[contains(@class, 'titles')]//tr[contains(@class, 'syn')]/td/text()", "shortNames" to "//div[contains(@class, 'titles')]//tr[contains(@class, 'short')]/td/text()", @@ -171,17 +172,20 @@ public class AnidbAnimeConverter( return hashSetOf(metaDataProviderConfig.buildAnimeLink(id)) } - private fun extractRelatedAnime(data: ExtractionResult): HashSet { + private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet { if (data.notFound("relatedAnime")) { return hashSetOf() } - return data.listNotNull("relatedAnime") - .asSequence() - .filterNot { it.contains("relation") } - .map { it.remove("/anime/") } - .distinct() - .map { metaDataProviderConfig.buildAnimeLink(it) } + //
... ...
Sequel
+ // Entries in "indirectly related" don't have a type. + return extractor.extractLabelledLinks( + snippets = data.listNotNull("relatedAnime"), + labelSelector = "//div[@class='type']/text()", + linkSelector = "//a/@href", + ).flatMap { entry -> entry.links.map { it to entry.label } } + .filterNot { (link, _) -> link.contains("relation") } + .map { (link, label) -> RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(link.remove("/anime/")), label) } .toHashSet() } diff --git a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverter.kt b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverter.kt index 73771c90..0d1f697d 100644 --- a/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverter.kt +++ b/anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverter.kt @@ -17,6 +17,7 @@ import io.github.manamiproject.modb.core.extensions.EMPTY import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank import io.github.manamiproject.modb.core.extractor.DataExtractor import io.github.manamiproject.modb.core.extractor.ExtractionResult +import io.github.manamiproject.modb.core.extractor.extractLabelledLinks import io.github.manamiproject.modb.core.extractor.XmlDataExtractor import kotlinx.coroutines.withContext import java.net.URI @@ -50,7 +51,7 @@ public class AnidbApiAnimeConverter( "mainTitle" to "//anime/titles/title[@type='main']/text()", "otherTitles" to "//anime/titles/title[@type!='main']/text()", "tags" to "//anime/tags/tag/name/text()", - "relatedAnime" to "//anime/relatedanime/anime/@id", + "relatedAnime" to "//anime/relatedanime/anime/html()", "episodeLengths" to "//anime/episodes/episode/length/text()", "score" to "//anime/ratings/permanent/text()", )) @@ -169,13 +170,17 @@ public class AnidbApiAnimeConverter( .toHashSet() } - private fun extractRelatedAnime(data: ExtractionResult): HashSet { + private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet { if (data.notFound("relatedAnime")) { return hashSetOf() } - return data.listNotNull("relatedAnime") - .map { metaDataProviderConfig.buildAnimeLink(it.trim()) } + // Cowboy Bebop: Tengoku no Tobira + return extractor.extractLabelledLinks( + snippets = data.listNotNull("relatedAnime"), + labelSelector = "//anime/@type", + linkSelector = "//anime/@id", + ).flatMap { entry -> entry.links.map { RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(it.trim()), entry.label) } } .toHashSet() } diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverterTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverterTest.kt index 8098276b..e3ff9113 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverterTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverterTest.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.anidb +import io.github.manamiproject.modb.core.anime.RelatedAnime import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.* import io.github.manamiproject.modb.core.anime.AnimeStatus.* import io.github.manamiproject.modb.core.anime.AnimeType.* @@ -536,15 +537,15 @@ internal class AnidbAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactlyInAnyOrder( - URI("https://anidb.net/anime/405"), - URI("https://anidb.net/anime/368"), - URI("https://anidb.net/anime/2850"), - URI("https://anidb.net/anime/4576"), - URI("https://anidb.net/anime/2995"), - URI("https://anidb.net/anime/6141"), - URI("https://anidb.net/anime/367"), - URI("https://anidb.net/anime/2996"), - URI("https://anidb.net/anime/6393"), + RelatedAnime(URI("https://anidb.net/anime/405"), "Sequel"), + RelatedAnime(URI("https://anidb.net/anime/368"), "Side story"), + RelatedAnime(URI("https://anidb.net/anime/2850")), + RelatedAnime(URI("https://anidb.net/anime/4576"), "Prequel"), + RelatedAnime(URI("https://anidb.net/anime/2995")), + RelatedAnime(URI("https://anidb.net/anime/6141"), "Side story"), + RelatedAnime(URI("https://anidb.net/anime/367"), "Side story"), + RelatedAnime(URI("https://anidb.net/anime/2996")), + RelatedAnime(URI("https://anidb.net/anime/6393"), "Side story"), ) } } diff --git a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverterTest.kt b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverterTest.kt index ec90717d..976da2e7 100644 --- a/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverterTest.kt +++ b/anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverterTest.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.anidb +import io.github.manamiproject.modb.core.anime.RelatedAnime import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.SPRING import io.github.manamiproject.modb.core.anime.AnimeStatus.FINISHED import io.github.manamiproject.modb.core.anime.AnimeType.TV @@ -43,7 +44,7 @@ internal class AnidbApiAnimeConverterTest { assertThat(result.sources).containsExactly(URI("https://anidb.net/anime/23")) assertThat(result.picture).isEqualTo(URI("https://cdn.anidb.net/images/main/221595.jpg")) assertThat(result.thumbnail).isEqualTo(URI("https://cdn.anidb.net/images/main/221595.jpg-thumb.jpg")) - assertThat(result.relatedAnime).containsExactly(URI("https://anidb.net/anime/219")) + assertThat(result.relatedAnime).containsExactly(RelatedAnime(URI("https://anidb.net/anime/219"), "Side Story")) assertThat(result.synonyms).contains("Kaubojus Bibopas") assertThat(result.tags).contains("bounty hunter", "detective") } diff --git a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverter.kt b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverter.kt index eb421a62..924940dc 100644 --- a/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverter.kt +++ b/anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverter.kt @@ -46,7 +46,7 @@ public class AnilistAnimeConverter( "id" to "$.data.Media.id", "status" to "$.data.Media.status", "tags" to "$.data.Media.tags.*.name", - "relatedAnime" to "$.data.Media.relations.edges.*.node", + "relatedAnime" to "$.data.Media.relations.edges", "score" to "$.data.Media.meanScore", "creators" to "$.data.Media.studios.edges", )) @@ -126,11 +126,17 @@ public class AnilistAnimeConverter( return hashSetOf(metaDataProviderConfig.buildAnimeLink(data.string("id").trim())) } - private fun extractRelatedAnime(data: ExtractionResult): HashSet { - return data.listNotNull>("relatedAnime") - .filter { it["type"] == "ANIME" } - .mapNotNull { it["id"] } - .map { metaDataProviderConfig.buildAnimeLink(it.toString().trim()) } + private fun extractRelatedAnime(data: ExtractionResult): HashSet { + return data.listNotNull>("relatedAnime") + .mapNotNull { edge -> + val node = edge["node"] as? Map<*, *> ?: return@mapNotNull null + if (node["type"] != "ANIME") return@mapNotNull null + val id = node["id"] ?: return@mapNotNull null + RelatedAnime.of( + uri = metaDataProviderConfig.buildAnimeLink(id.toString().trim()), + relation = edge["relationType"]?.toString(), + ) + } .toHashSet() } diff --git a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverterTest.kt b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverterTest.kt index b304483a..a81a6b72 100644 --- a/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverterTest.kt +++ b/anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverterTest.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.anilist +import io.github.manamiproject.modb.core.anime.RelatedAnime import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.* import io.github.manamiproject.modb.core.anime.AnimeStatus.* import io.github.manamiproject.modb.core.anime.AnimeType.* @@ -410,9 +411,9 @@ internal class AnilistAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactlyInAnyOrder( - URI("https://anilist.co/anime/107298"), - URI("https://anilist.co/anime/116147"), - URI("https://anilist.co/anime/97857"), + RelatedAnime(URI("https://anilist.co/anime/107298"), "SIDE_STORY"), + RelatedAnime(URI("https://anilist.co/anime/116147"), "SEQUEL"), + RelatedAnime(URI("https://anilist.co/anime/97857"), "PREQUEL"), ) } } @@ -434,7 +435,7 @@ internal class AnilistAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactly( - URI("https://anilist.co/anime/2337") + RelatedAnime(URI("https://anilist.co/anime/2337"), "ALTERNATIVE") ) } } @@ -456,18 +457,18 @@ internal class AnilistAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactlyInAnyOrder( - URI("https://anilist.co/anime/100148"), - URI("https://anilist.co/anime/1081"), - URI("https://anilist.co/anime/1301"), - URI("https://anilist.co/anime/1302"), - URI("https://anilist.co/anime/1491"), - URI("https://anilist.co/anime/1645"), - URI("https://anilist.co/anime/1676"), - URI("https://anilist.co/anime/1706"), - URI("https://anilist.co/anime/17269"), - URI("https://anilist.co/anime/2202"), - URI("https://anilist.co/anime/2203"), - URI("https://anilist.co/anime/2470"), + RelatedAnime(URI("https://anilist.co/anime/100148"), "ALTERNATIVE"), + RelatedAnime(URI("https://anilist.co/anime/1081"), "SIDE_STORY"), + RelatedAnime(URI("https://anilist.co/anime/1301"), "PREQUEL"), + RelatedAnime(URI("https://anilist.co/anime/1302"), "CHARACTER"), + RelatedAnime(URI("https://anilist.co/anime/1491"), "CHARACTER"), + RelatedAnime(URI("https://anilist.co/anime/1645"), "SEQUEL"), + RelatedAnime(URI("https://anilist.co/anime/1676"), "SIDE_STORY"), + RelatedAnime(URI("https://anilist.co/anime/1706"), "ALTERNATIVE"), + RelatedAnime(URI("https://anilist.co/anime/17269"), "ALTERNATIVE"), + RelatedAnime(URI("https://anilist.co/anime/2202"), "ALTERNATIVE"), + RelatedAnime(URI("https://anilist.co/anime/2203"), "ALTERNATIVE"), + RelatedAnime(URI("https://anilist.co/anime/2470"), "SIDE_STORY"), ) } } diff --git a/anime-planet/src/main/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverter.kt b/anime-planet/src/main/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverter.kt index 038d1805..50d87213 100644 --- a/anime-planet/src/main/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverter.kt +++ b/anime-planet/src/main/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverter.kt @@ -9,6 +9,7 @@ import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank import io.github.manamiproject.modb.core.extensions.remove import io.github.manamiproject.modb.core.extractor.DataExtractor import io.github.manamiproject.modb.core.extractor.ExtractionResult +import io.github.manamiproject.modb.core.extractor.extractLabelledLinks import io.github.manamiproject.modb.core.extractor.JsonDataExtractor import io.github.manamiproject.modb.core.extractor.XmlDataExtractor import io.github.manamiproject.modb.core.anime.* @@ -51,7 +52,7 @@ public class AnimePlanetAnimeConverter( "seasonYear" to "//section[@class='pure-g entryBar']//span[@class='iconYear']/following-sibling::*/text()", "source" to "//link[@rel='canonical']/@href", "alternativeTitle" to "//h2[@class='aka']/text()", - "relatedAnime" to "//div[@id='tabs--relations--anime']/div//a/@href", + "relatedAnime" to "//div[@id='tabs--relations--anime']/div//a/html()", "tags" to "//div[contains(@class, 'tags')]//a/text()", "studios" to "//a[contains(@href, '/studios/')]/text()", )) @@ -266,15 +267,19 @@ public class AnimePlanetAnimeConverter( return tags } - private fun extractRelatedAnime(data: ExtractionResult): HashSet { - return if (data.notFound("relatedAnime")) { - hashSetOf() - } else { - data.listNotNull("relatedAnime") - .map { it.remove("/anime/") } - .map { metaDataProviderConfig.buildAnimeLink(it) } - .toHashSet() + private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet { + if (data.notFound("relatedAnime")) { + return hashSetOf() } + + //
... Side Story + // The subtitle is optional. + return xmlExtractor.extractLabelledLinks( + snippets = data.listNotNull("relatedAnime"), + labelSelector = "//span[contains(@class, 'RelatedEntry__subtitle')]/text()", + linkSelector = "//a/@href", + ).flatMap { entry -> entry.links.map { RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(it.remove("/anime/")), entry.label) } } + .toHashSet() } private fun extractScore(jsonldData: ExtractionResult): MetaDataProviderScore { diff --git a/anime-planet/src/test/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverterTest.kt b/anime-planet/src/test/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverterTest.kt index a399d66f..fffdc25a 100644 --- a/anime-planet/src/test/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverterTest.kt +++ b/anime-planet/src/test/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverterTest.kt @@ -1,5 +1,6 @@ package io.github.manamiproject.modb.animeplanet +import io.github.manamiproject.modb.core.anime.RelatedAnime import io.github.manamiproject.modb.core.anime.AnimeSeason import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.* import io.github.manamiproject.modb.core.anime.AnimeStatus.* @@ -1020,13 +1021,13 @@ internal class AnimePlanetAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactlyInAnyOrder( - URI("https://anime-planet.com/anime/fullmetal-alchemist"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-4-koma-theater"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-specials"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-premium-collection"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-reflections"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-the-movie-conqueror-of-shamballa"), - URI("https://anime-planet.com/anime/fullmetal-alchemist-the-sacred-star-of-milos"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist"), "Other Franchise"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-4-koma-theater"), "Omake"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-specials"), "Side Story"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-premium-collection"), "Other Franchise"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-reflections"), "Other Franchise"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-the-movie-conqueror-of-shamballa"), "Other Franchise"), + RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-the-sacred-star-of-milos")), ) } } @@ -1049,12 +1050,12 @@ internal class AnimePlanetAnimeConverterTest { // then assertThat(result.relatedAnime).containsExactlyInAnyOrder( - URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-entertainment-district-arc"), - URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-movie-mugen-train"), - URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-mugen-train"), - URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-swordsmith-village-arc"), - URI("https://anime-planet.com/anime/kimetsu-gakuen-monogatari-kimetsu-no-utage-tokubetsu-hen"), - URI("https://anime-planet.com/anime/kimetsu-gakuen-valentine-hen"), + RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-entertainment-district-arc")), + RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-movie-mugen-train"), "Sequel"), + RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-mugen-train"), "Sequel"), + RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-swordsmith-village-arc")), + RelatedAnime(URI("https://anime-planet.com/anime/kimetsu-gakuen-monogatari-kimetsu-no-utage-tokubetsu-hen")), + RelatedAnime(URI("https://anime-planet.com/anime/kimetsu-gakuen-valentine-hen")), ) } } diff --git a/animenewsnetwork/src/main/kotlin/io/github/manamiproject/AnimenewsnetworkAnimeConverter.kt b/animenewsnetwork/src/main/kotlin/io/github/manamiproject/AnimenewsnetworkAnimeConverter.kt index 4be03d00..8b6ef3ca 100644 --- a/animenewsnetwork/src/main/kotlin/io/github/manamiproject/AnimenewsnetworkAnimeConverter.kt +++ b/animenewsnetwork/src/main/kotlin/io/github/manamiproject/AnimenewsnetworkAnimeConverter.kt @@ -56,7 +56,7 @@ public class AnimenewsnetworkAnimeConverter( "synonyms" to "//div[@id='infotype-2']//div", "genres" to "//div[@id='infotype-30']//a/text()", "themes" to "//div[@id='infotype-31']//a/text()", - "relatedAnime" to "//div[@id='infotype-related']//a/@href", + "relatedAnime" to "//div[@id='infotype-related']/html()", "duration" to "//div[@id='infotype-4']//span/text()", "year_list_all" to "//div[@id='infotype-7']//div//text()", )) @@ -275,17 +275,27 @@ public class AnimenewsnetworkAnimeConverter( return genres.union(themes).toHashSet() } - private fun extractRelatedAnime(data: ExtractionResult): HashSet { - val entries = if (data.notFound("relatedAnime")) { - hashSetOf() - } else { - data.listNotNull("relatedAnime") + /** + * The relation is plain text next to the link: + * + `[ sequel of Clannad (TV) ]` (header) → `sequel of` + * + `Frieren (TV 2) (sequel)
` (list) → `sequel` + */ + private fun extractRelatedAnime(data: ExtractionResult): HashSet { + if (data.notFound("relatedAnime")) { + return hashSetOf() } - return entries.filter { it.contains("anime.php") } - .map { it.substringAfterLast('=') } - .map { metaDataProviderConfig.buildAnimeLink(it) } - .toHashSet() + return data.listNotNull("relatedAnime").flatMap { html -> + relatedAnimeLinkRegex.findAll(html).map { match -> + val textBefore = html.substring(0, match.range.first).substringAfterLast('>') + val label = if (textBefore.contains('[') || textBefore.trimStart().startsWith(',')) { + textBefore.substringAfterLast('[').trimStart().removePrefix(",") + } else { + relationInBracketsRegex.find(match.groups["after"]?.value ?: EMPTY)?.groups?.get("label")?.value + } + RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(match.groups["id"]!!.value), label) + }.toList() + }.toHashSet() } private fun extractDuration(data: ExtractionResult): Duration { @@ -440,6 +450,8 @@ public class AnimenewsnetworkAnimeConverter( season = UNDEFINED, ) + private val relatedAnimeLinkRegex = """]*href="[^"]*anime\.php\?id=(?\d+)"[^>]*>[^<]*(?[^<]*)""".toRegex() + private val relationInBracketsRegex = """^\s*\((?