Add the relation type to relatedAnime

anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverter.kt

@@ -9,6 +9,7 @@ import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank
 import io.github.manamiproject.modb.core.extensions.remove
 import io.github.manamiproject.modb.core.extractor.DataExtractor
 import io.github.manamiproject.modb.core.extractor.ExtractionResult
+import io.github.manamiproject.modb.core.extractor.extractLabelledLinks
 import io.github.manamiproject.modb.core.extractor.XmlDataExtractor
 import io.github.manamiproject.modb.core.anime.*
 import io.github.manamiproject.modb.core.anime.AnimeMedia.NO_PICTURE
@@ -53,7 +54,7 @@ public class AnidbAnimeConverter(
             "season" to "//tr[contains(@class, 'season')]//td[contains(@class, 'value')]/text()",
             "startDate" to "//tr[contains(@class, 'year')]//td[contains(@class, 'value')]//span[contains(@itemprop, 'startDate')]/@content",
             "datePublished" to "//tr[contains(@class, 'year')]//td[contains(@class, 'value')]//span[contains(@itemprop, 'datePublished')]/@content",
-            "relatedAnime" to "//div[contains(@class, 'directly_related')]//a/@href",
+            "relatedAnime" to "//div[contains(@class, 'directly_related')]//div[contains(@class, 'stripe')]/html()",
             "alternateNames" to "//label[contains(@itemprop, 'alternateName')]/text()",
             "synonymsList" to "//div[contains(@class, 'titles')]//tr[contains(@class, 'syn')]/td/text()",
             "shortNames" to "//div[contains(@class, 'titles')]//tr[contains(@class, 'short')]/td/text()",
@@ -171,17 +172,20 @@ public class AnidbAnimeConverter(
         return hashSetOf(metaDataProviderConfig.buildAnimeLink(id))
     }
 
-    private fun extractRelatedAnime(data: ExtractionResult): HashSet<URI> {
+    private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet<RelatedAnime> {
         if (data.notFound("relatedAnime")) {
             return hashSetOf()
         }
 
-        return data.listNotNull<String>("relatedAnime")
-            .asSequence()
-            .filterNot { it.contains("relation") }
-            .map { it.remove("/anime/") }
-            .distinct()
-            .map { metaDataProviderConfig.buildAnimeLink(it) }
+        // <div class="g_bubble stripe"> ... <a href="/anime/405"> ... <div class="type">Sequel</div>
+        // Entries in "indirectly related" don't have a type.
+        return extractor.extractLabelledLinks(
+            snippets = data.listNotNull<String>("relatedAnime"),
+            labelSelector = "//div[@class='type']/text()",
+            linkSelector = "//a/@href",
+        ).flatMap { entry -> entry.links.map { it to entry.label } }
+            .filterNot { (link, _) -> link.contains("relation") }
+            .map { (link, label) -> RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(link.remove("/anime/")), label) }
             .toHashSet()
     }
 

anidb/src/main/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverter.kt

@@ -17,6 +17,7 @@ import io.github.manamiproject.modb.core.extensions.EMPTY
 import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank
 import io.github.manamiproject.modb.core.extractor.DataExtractor
 import io.github.manamiproject.modb.core.extractor.ExtractionResult
+import io.github.manamiproject.modb.core.extractor.extractLabelledLinks
 import io.github.manamiproject.modb.core.extractor.XmlDataExtractor
 import kotlinx.coroutines.withContext
 import java.net.URI
@@ -50,7 +51,7 @@ public class AnidbApiAnimeConverter(
             "mainTitle" to "//anime/titles/title[@type='main']/text()",
             "otherTitles" to "//anime/titles/title[@type!='main']/text()",
             "tags" to "//anime/tags/tag/name/text()",
-            "relatedAnime" to "//anime/relatedanime/anime/@id",
+            "relatedAnime" to "//anime/relatedanime/anime/html()",
             "episodeLengths" to "//anime/episodes/episode/length/text()",
             "score" to "//anime/ratings/permanent/text()",
         ))
@@ -169,13 +170,17 @@ public class AnidbApiAnimeConverter(
             .toHashSet()
     }
 
-    private fun extractRelatedAnime(data: ExtractionResult): HashSet<URI> {
+    private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet<RelatedAnime> {
         if (data.notFound("relatedAnime")) {
             return hashSetOf()
         }
 
-        return data.listNotNull<String>("relatedAnime")
-            .map { metaDataProviderConfig.buildAnimeLink(it.trim()) }
+        // <anime id="219" type="Side Story">Cowboy Bebop: Tengoku no Tobira</anime>
+        return extractor.extractLabelledLinks(
+            snippets = data.listNotNull<String>("relatedAnime"),
+            labelSelector = "//anime/@type",
+            linkSelector = "//anime/@id",
+        ).flatMap { entry -> entry.links.map { RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(it.trim()), entry.label) } }
             .toHashSet()
     }
 

anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbAnimeConverterTest.kt

@@ -1,5 +1,6 @@
 package io.github.manamiproject.modb.anidb
 
+import io.github.manamiproject.modb.core.anime.RelatedAnime
 import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.*
 import io.github.manamiproject.modb.core.anime.AnimeStatus.*
 import io.github.manamiproject.modb.core.anime.AnimeType.*
@@ -536,15 +537,15 @@ internal class AnidbAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactlyInAnyOrder(
-                    URI("https://anidb.net/anime/405"),
-                    URI("https://anidb.net/anime/368"),
-                    URI("https://anidb.net/anime/2850"),
-                    URI("https://anidb.net/anime/4576"),
-                    URI("https://anidb.net/anime/2995"),
-                    URI("https://anidb.net/anime/6141"),
-                    URI("https://anidb.net/anime/367"),
-                    URI("https://anidb.net/anime/2996"),
-                    URI("https://anidb.net/anime/6393"),
+                    RelatedAnime(URI("https://anidb.net/anime/405"), "Sequel"),
+                    RelatedAnime(URI("https://anidb.net/anime/368"), "Side story"),
+                    RelatedAnime(URI("https://anidb.net/anime/2850")),
+                    RelatedAnime(URI("https://anidb.net/anime/4576"), "Prequel"),
+                    RelatedAnime(URI("https://anidb.net/anime/2995")),
+                    RelatedAnime(URI("https://anidb.net/anime/6141"), "Side story"),
+                    RelatedAnime(URI("https://anidb.net/anime/367"), "Side story"),
+                    RelatedAnime(URI("https://anidb.net/anime/2996")),
+                    RelatedAnime(URI("https://anidb.net/anime/6393"), "Side story"),
                 )
             }
         }

anidb/src/test/kotlin/io/github/manamiproject/modb/anidb/AnidbApiAnimeConverterTest.kt

@@ -1,5 +1,6 @@
 package io.github.manamiproject.modb.anidb
 
+import io.github.manamiproject.modb.core.anime.RelatedAnime
 import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.SPRING
 import io.github.manamiproject.modb.core.anime.AnimeStatus.FINISHED
 import io.github.manamiproject.modb.core.anime.AnimeType.TV
@@ -43,7 +44,7 @@ internal class AnidbApiAnimeConverterTest {
                 assertThat(result.sources).containsExactly(URI("https://anidb.net/anime/23"))
                 assertThat(result.picture).isEqualTo(URI("https://cdn.anidb.net/images/main/221595.jpg"))
                 assertThat(result.thumbnail).isEqualTo(URI("https://cdn.anidb.net/images/main/221595.jpg-thumb.jpg"))
-                assertThat(result.relatedAnime).containsExactly(URI("https://anidb.net/anime/219"))
+                assertThat(result.relatedAnime).containsExactly(RelatedAnime(URI("https://anidb.net/anime/219"), "Side Story"))
                 assertThat(result.synonyms).contains("Kaubojus Bibopas")
                 assertThat(result.tags).contains("bounty hunter", "detective")
             }

anilist/src/main/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverter.kt

@@ -46,7 +46,7 @@ public class AnilistAnimeConverter(
             "id" to "$.data.Media.id",
             "status" to "$.data.Media.status",
             "tags" to "$.data.Media.tags.*.name",
-            "relatedAnime" to "$.data.Media.relations.edges.*.node",
+            "relatedAnime" to "$.data.Media.relations.edges",
             "score" to "$.data.Media.meanScore",
             "creators" to "$.data.Media.studios.edges",
         ))
@@ -126,11 +126,17 @@ public class AnilistAnimeConverter(
         return hashSetOf(metaDataProviderConfig.buildAnimeLink(data.string("id").trim()))
     }
 
-    private fun extractRelatedAnime(data: ExtractionResult): HashSet<URI> {
-        return data.listNotNull<LinkedHashMap<String, Any>>("relatedAnime")
-            .filter { it["type"] == "ANIME" }
-            .mapNotNull { it["id"] }
-            .map { metaDataProviderConfig.buildAnimeLink(it.toString().trim()) }
+    private fun extractRelatedAnime(data: ExtractionResult): HashSet<RelatedAnime> {
+        return data.listNotNull<Map<String, Any>>("relatedAnime")
+            .mapNotNull { edge ->
+                val node = edge["node"] as? Map<*, *> ?: return@mapNotNull null
+                if (node["type"] != "ANIME") return@mapNotNull null
+                val id = node["id"] ?: return@mapNotNull null
+                RelatedAnime.of(
+                    uri = metaDataProviderConfig.buildAnimeLink(id.toString().trim()),
+                    relation = edge["relationType"]?.toString(),
+                )
+            }
             .toHashSet()
     }
 

anilist/src/test/kotlin/io/github/manamiproject/modb/anilist/AnilistAnimeConverterTest.kt

@@ -1,5 +1,6 @@
 package io.github.manamiproject.modb.anilist
 
+import io.github.manamiproject.modb.core.anime.RelatedAnime
 import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.*
 import io.github.manamiproject.modb.core.anime.AnimeStatus.*
 import io.github.manamiproject.modb.core.anime.AnimeType.*
@@ -410,9 +411,9 @@ internal class AnilistAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactlyInAnyOrder(
-                    URI("https://anilist.co/anime/107298"),
-                    URI("https://anilist.co/anime/116147"),
-                    URI("https://anilist.co/anime/97857"),
+                    RelatedAnime(URI("https://anilist.co/anime/107298"), "SIDE_STORY"),
+                    RelatedAnime(URI("https://anilist.co/anime/116147"), "SEQUEL"),
+                    RelatedAnime(URI("https://anilist.co/anime/97857"), "PREQUEL"),
                 )
             }
         }
@@ -434,7 +435,7 @@ internal class AnilistAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactly(
-                    URI("https://anilist.co/anime/2337")
+                    RelatedAnime(URI("https://anilist.co/anime/2337"), "ALTERNATIVE")
                 )
             }
         }
@@ -456,18 +457,18 @@ internal class AnilistAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactlyInAnyOrder(
-                    URI("https://anilist.co/anime/100148"),
-                    URI("https://anilist.co/anime/1081"),
-                    URI("https://anilist.co/anime/1301"),
-                    URI("https://anilist.co/anime/1302"),
-                    URI("https://anilist.co/anime/1491"),
-                    URI("https://anilist.co/anime/1645"),
-                    URI("https://anilist.co/anime/1676"),
-                    URI("https://anilist.co/anime/1706"),
-                    URI("https://anilist.co/anime/17269"),
-                    URI("https://anilist.co/anime/2202"),
-                    URI("https://anilist.co/anime/2203"),
-                    URI("https://anilist.co/anime/2470"),
+                    RelatedAnime(URI("https://anilist.co/anime/100148"), "ALTERNATIVE"),
+                    RelatedAnime(URI("https://anilist.co/anime/1081"), "SIDE_STORY"),
+                    RelatedAnime(URI("https://anilist.co/anime/1301"), "PREQUEL"),
+                    RelatedAnime(URI("https://anilist.co/anime/1302"), "CHARACTER"),
+                    RelatedAnime(URI("https://anilist.co/anime/1491"), "CHARACTER"),
+                    RelatedAnime(URI("https://anilist.co/anime/1645"), "SEQUEL"),
+                    RelatedAnime(URI("https://anilist.co/anime/1676"), "SIDE_STORY"),
+                    RelatedAnime(URI("https://anilist.co/anime/1706"), "ALTERNATIVE"),
+                    RelatedAnime(URI("https://anilist.co/anime/17269"), "ALTERNATIVE"),
+                    RelatedAnime(URI("https://anilist.co/anime/2202"), "ALTERNATIVE"),
+                    RelatedAnime(URI("https://anilist.co/anime/2203"), "ALTERNATIVE"),
+                    RelatedAnime(URI("https://anilist.co/anime/2470"), "SIDE_STORY"),
                 )
             }
         }

anime-planet/src/main/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverter.kt

@@ -9,6 +9,7 @@ import io.github.manamiproject.modb.core.extensions.neitherNullNorBlank
 import io.github.manamiproject.modb.core.extensions.remove
 import io.github.manamiproject.modb.core.extractor.DataExtractor
 import io.github.manamiproject.modb.core.extractor.ExtractionResult
+import io.github.manamiproject.modb.core.extractor.extractLabelledLinks
 import io.github.manamiproject.modb.core.extractor.JsonDataExtractor
 import io.github.manamiproject.modb.core.extractor.XmlDataExtractor
 import io.github.manamiproject.modb.core.anime.*
@@ -51,7 +52,7 @@ public class AnimePlanetAnimeConverter(
             "seasonYear" to "//section[@class='pure-g entryBar']//span[@class='iconYear']/following-sibling::*/text()",
             "source" to "//link[@rel='canonical']/@href",
             "alternativeTitle" to "//h2[@class='aka']/text()",
-            "relatedAnime" to "//div[@id='tabs--relations--anime']/div//a/@href",
+            "relatedAnime" to "//div[@id='tabs--relations--anime']/div//a/html()",
             "tags" to "//div[contains(@class, 'tags')]//a/text()",
             "studios" to "//a[contains(@href, '/studios/')]/text()",
         ))
@@ -266,15 +267,19 @@ public class AnimePlanetAnimeConverter(
         return tags
     }
 
-    private fun extractRelatedAnime(data: ExtractionResult): HashSet<URI> {
-        return if (data.notFound("relatedAnime")) {
-            hashSetOf()
-        } else {
-            data.listNotNull<String>("relatedAnime")
-                .map { it.remove("/anime/") }
-                .map { metaDataProviderConfig.buildAnimeLink(it) }
-                .toHashSet()
+    private suspend fun extractRelatedAnime(data: ExtractionResult): HashSet<RelatedAnime> {
+        if (data.notFound("relatedAnime")) {
+            return hashSetOf()
         }
+
+        // <a href="/anime/..." class="RelatedEntry ..."> ... <span class="RelatedEntry__subtitle ...">Side Story</span>
+        // The subtitle is optional.
+        return xmlExtractor.extractLabelledLinks(
+            snippets = data.listNotNull<String>("relatedAnime"),
+            labelSelector = "//span[contains(@class, 'RelatedEntry__subtitle')]/text()",
+            linkSelector = "//a/@href",
+        ).flatMap { entry -> entry.links.map { RelatedAnime.of(metaDataProviderConfig.buildAnimeLink(it.remove("/anime/")), entry.label) } }
+            .toHashSet()
     }
 
     private fun extractScore(jsonldData: ExtractionResult): MetaDataProviderScore {

anime-planet/src/test/kotlin/io/github/manamiproject/modb/animeplanet/AnimePlanetAnimeConverterTest.kt

@@ -1,5 +1,6 @@
 package io.github.manamiproject.modb.animeplanet
 
+import io.github.manamiproject.modb.core.anime.RelatedAnime
 import io.github.manamiproject.modb.core.anime.AnimeSeason
 import io.github.manamiproject.modb.core.anime.AnimeSeason.Season.*
 import io.github.manamiproject.modb.core.anime.AnimeStatus.*
@@ -1020,13 +1021,13 @@ internal class AnimePlanetAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactlyInAnyOrder(
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-4-koma-theater"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-specials"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-premium-collection"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-reflections"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-the-movie-conqueror-of-shamballa"),
-                    URI("https://anime-planet.com/anime/fullmetal-alchemist-the-sacred-star-of-milos"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist"), "Other Franchise"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-4-koma-theater"), "Omake"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-brotherhood-specials"), "Side Story"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-premium-collection"), "Other Franchise"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-reflections"), "Other Franchise"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-the-movie-conqueror-of-shamballa"), "Other Franchise"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/fullmetal-alchemist-the-sacred-star-of-milos")),
                 )
             }
         }
@@ -1049,12 +1050,12 @@ internal class AnimePlanetAnimeConverterTest {
 
                 // then
                 assertThat(result.relatedAnime).containsExactlyInAnyOrder(
-                    URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-entertainment-district-arc"),
-                    URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-movie-mugen-train"),
-                    URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-mugen-train"),
-                    URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-swordsmith-village-arc"),
-                    URI("https://anime-planet.com/anime/kimetsu-gakuen-monogatari-kimetsu-no-utage-tokubetsu-hen"),
-                    URI("https://anime-planet.com/anime/kimetsu-gakuen-valentine-hen"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-entertainment-district-arc")),
+                    RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-movie-mugen-train"), "Sequel"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-mugen-train"), "Sequel"),
+                    RelatedAnime(URI("https://anime-planet.com/anime/demon-slayer-kimetsu-no-yaiba-swordsmith-village-arc")),
+                    RelatedAnime(URI("https://anime-planet.com/anime/kimetsu-gakuen-monogatari-kimetsu-no-utage-tokubetsu-hen")),
+                    RelatedAnime(URI("https://anime-planet.com/anime/kimetsu-gakuen-valentine-hen")),
                 )
             }
         }