Skip to content

Commit ff2dc78

Browse files
PriveeteePriveetee
committed
fix: repair imported subscription avatars
1 parent 08a0da5 commit ff2dc78

4 files changed

Lines changed: 218 additions & 108 deletions

File tree

‎server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
package dev.typetype.server.services
2+
3+
import dev.typetype.server.db.tables.FavoritesTable
4+
import dev.typetype.server.db.tables.HistoryTable
5+
import dev.typetype.server.db.tables.PlaylistVideosTable
6+
import dev.typetype.server.db.tables.SubscriptionsTable
7+
import dev.typetype.server.db.tables.WatchLaterTable
8+
import dev.typetype.server.models.SubscriptionItem
9+
import org.jetbrains.exposed.v1.core.Column
10+
import org.jetbrains.exposed.v1.core.ResultRow
11+
import org.jetbrains.exposed.v1.core.SortOrder
12+
import org.jetbrains.exposed.v1.core.and
13+
import org.jetbrains.exposed.v1.core.eq
14+
import org.jetbrains.exposed.v1.core.inList
15+
import org.jetbrains.exposed.v1.core.neq
16+
import org.jetbrains.exposed.v1.jdbc.selectAll
17+
import org.jetbrains.exposed.v1.jdbc.update
18+
19+
object SubscriptionAvatarRepairer {
20+
fun repair(userId: String, items: List<SubscriptionItem>): List<SubscriptionItem> {
21+
val avatars = recoverableAvatars(
22+
userId = userId,
23+
items = items,
24+
maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST,
25+
maxSourceRows = MAX_AVATAR_SOURCE_ROWS,
26+
)
27+
avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) }
28+
return items.withAvatars(avatars)
29+
}
30+
31+
fun repairImported(userId: String, items: List<SubscriptionItem>): List<SubscriptionItem> {
32+
val avatars = recoverableAvatars(
33+
userId = userId,
34+
items = items,
35+
maxRepairs = MAX_IMPORTED_AVATAR_REPAIR,
36+
maxSourceRows = MAX_IMPORTED_AVATAR_SOURCE_ROWS,
37+
)
38+
avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) }
39+
return items.withAvatars(avatars)
40+
}
41+
42+
fun resolve(userId: String, items: List<SubscriptionItem>): List<SubscriptionItem> =
43+
items.withAvatars(
44+
recoverableAvatars(
45+
userId = userId,
46+
items = items,
47+
maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST,
48+
maxSourceRows = MAX_AVATAR_SOURCE_ROWS,
49+
),
50+
)
51+
52+
private fun recoverableAvatars(
53+
userId: String,
54+
items: List<SubscriptionItem>,
55+
maxRepairs: Int,
56+
maxSourceRows: Int,
57+
): Map<String, String> {
58+
val candidateUrls = items.filter { it.avatarUrl.isBlank() }
59+
.map { it.channelUrl }
60+
.distinct()
61+
if (candidateUrls.isEmpty()) return emptyMap()
62+
val avatars = knownAvatars(userId, candidateUrls, maxSourceRows)
63+
.entries
64+
.take(maxRepairs)
65+
.associate { it.toPair() }
66+
return avatars
67+
}
68+
69+
private fun List<SubscriptionItem>.withAvatars(avatars: Map<String, String>): List<SubscriptionItem> =
70+
if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item }
71+
72+
private fun knownAvatars(
73+
userId: String,
74+
channelUrls: List<String>,
75+
maxSourceRows: Int,
76+
): Map<String, String> {
77+
val avatars = linkedMapOf<String, String>()
78+
historyAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) }
79+
playlistAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) }
80+
watchLaterAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) }
81+
favoriteAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) }
82+
return avatars
83+
}
84+
85+
private fun historyAvatars(
86+
userId: String,
87+
channelUrls: List<String>,
88+
maxSourceRows: Int,
89+
): Map<String, String> = HistoryTable.selectAll()
90+
.where {
91+
avatarSourceFilter(
92+
userId,
93+
channelUrls,
94+
HistoryTable.userId,
95+
HistoryTable.channelUrl,
96+
HistoryTable.channelAvatar,
97+
)
98+
}
99+
.orderBy(HistoryTable.watchedAt to SortOrder.DESC)
100+
.limit(maxSourceRows)
101+
.associateAvatarRows(HistoryTable.channelUrl, HistoryTable.channelAvatar)
102+
103+
private fun playlistAvatars(
104+
userId: String,
105+
channelUrls: List<String>,
106+
maxSourceRows: Int,
107+
): Map<String, String> = PlaylistVideosTable.selectAll()
108+
.where {
109+
avatarSourceFilter(
110+
userId,
111+
channelUrls,
112+
PlaylistVideosTable.userId,
113+
PlaylistVideosTable.channelUrl,
114+
PlaylistVideosTable.channelAvatar,
115+
)
116+
}
117+
.limit(maxSourceRows)
118+
.associateAvatarRows(PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar)
119+
120+
private fun watchLaterAvatars(
121+
userId: String,
122+
channelUrls: List<String>,
123+
maxSourceRows: Int,
124+
): Map<String, String> = WatchLaterTable.selectAll()
125+
.where {
126+
avatarSourceFilter(
127+
userId,
128+
channelUrls,
129+
WatchLaterTable.userId,
130+
WatchLaterTable.channelUrl,
131+
WatchLaterTable.channelAvatar,
132+
)
133+
}
134+
.orderBy(WatchLaterTable.addedAt to SortOrder.DESC)
135+
.limit(maxSourceRows)
136+
.associateAvatarRows(WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar)
137+
138+
private fun favoriteAvatars(
139+
userId: String,
140+
channelUrls: List<String>,
141+
maxSourceRows: Int,
142+
): Map<String, String> = FavoritesTable.selectAll()
143+
.where {
144+
avatarSourceFilter(
145+
userId,
146+
channelUrls,
147+
FavoritesTable.userId,
148+
FavoritesTable.channelUrl,
149+
FavoritesTable.channelAvatar,
150+
)
151+
}
152+
.orderBy(FavoritesTable.favoritedAt to SortOrder.DESC)
153+
.limit(maxSourceRows)
154+
.associateAvatarRows(FavoritesTable.channelUrl, FavoritesTable.channelAvatar)
155+
156+
private fun updateAvatar(userId: String, channelUrl: String, avatarUrl: String): Int = SubscriptionsTable.update({
157+
(SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) and (SubscriptionsTable.avatarUrl eq "")
158+
}) {
159+
it[SubscriptionsTable.avatarUrl] = avatarUrl
160+
}
161+
162+
private fun avatarSourceFilter(
163+
userId: String,
164+
channelUrls: List<String>,
165+
userColumn: Column<String>,
166+
urlColumn: Column<String>,
167+
avatarColumn: Column<String>,
168+
) = (userColumn eq userId) and (urlColumn inList channelUrls) and (avatarColumn neq "")
169+
170+
private fun Iterable<ResultRow>.associateAvatarRows(
171+
urlColumn: Column<String>,
172+
avatarColumn: Column<String>,
173+
): Map<String, String> = mapNotNull { row ->
174+
val avatar = row[avatarColumn].trim()
175+
if (avatar.isProxyableAvatar()) ChannelUrlCanonicalizer.canonicalize(row[urlColumn]) to avatar else null
176+
}.distinctBy { it.first }.toMap()
177+
178+
private fun String.isProxyableAvatar(): Boolean = startsWith("https://") || startsWith("http://")
179+
180+
private const val MAX_AVATAR_REPAIR_PER_REQUEST = 25
181+
private const val MAX_AVATAR_SOURCE_ROWS = 100
182+
private const val MAX_IMPORTED_AVATAR_REPAIR = 5_000
183+
private const val MAX_IMPORTED_AVATAR_SOURCE_ROWS = 20_000
184+
}

‎server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import dev.typetype.server.db.tables.HistoryTable
55
import dev.typetype.server.db.tables.PlaylistVideosTable
66
import dev.typetype.server.db.tables.SubscriptionsTable
77
import dev.typetype.server.db.tables.WatchLaterTable
8+
import dev.typetype.server.models.SubscriptionItem
89
import dev.typetype.server.services.ChannelUrlCanonicalizer
10+
import dev.typetype.server.services.SubscriptionAvatarRepairer
911
import org.jetbrains.exposed.v1.core.and
1012
import org.jetbrains.exposed.v1.core.eq
1113
import org.jetbrains.exposed.v1.core.neq
@@ -14,17 +16,24 @@ import org.jetbrains.exposed.v1.jdbc.update
1416

1517
object PortabilityMediaAvatarPropagation {
1618
fun propagate(userId: String): Int {
17-
val avatars = SubscriptionsTable.selectAll()
19+
val subscriptions = SubscriptionsTable.selectAll()
1820
.where {
19-
(SubscriptionsTable.userId eq userId) and
20-
(SubscriptionsTable.avatarUrl neq "")
21+
SubscriptionsTable.userId eq userId
2122
}
2223
.limit(MAX_SUBSCRIPTIONS)
2324
.map {
24-
ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) to
25-
it[SubscriptionsTable.avatarUrl]
25+
SubscriptionItem(
26+
channelUrl = it[SubscriptionsTable.channelUrl],
27+
name = it[SubscriptionsTable.name],
28+
avatarUrl = it[SubscriptionsTable.avatarUrl],
29+
subscribedAt = it[SubscriptionsTable.subscribedAt],
30+
)
31+
}
32+
val avatars = SubscriptionAvatarRepairer.repairImported(userId, subscriptions)
33+
.filter { it.avatarUrl.isNotBlank() }
34+
.associate {
35+
ChannelUrlCanonicalizer.canonicalize(it.channelUrl) to it.avatarUrl
2636
}
27-
.toMap()
2837
return avatars.entries.sumOf { (channelUrl, avatarUrl) ->
2938
HistoryTable.update({
3039
(HistoryTable.userId eq userId) and

‎server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt‎

Lines changed: 0 additions & 102 deletions
This file was deleted.

‎src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,25 @@ class SubscriptionsAvatarRepairServiceTest {
128128
assertEquals(WATCH_AVATAR_URL, storedFavoriteAvatar())
129129
}
130130

131+
@Test
132+
fun `import propagation repairs subscription avatars from media first`() = runTest {
133+
DatabaseFactory.query {
134+
SubscriptionsTable.insert {
135+
it[userId] = TEST_USER_ID
136+
it[channelUrl] = WATCH_CHANNEL_URL
137+
it[name] = "Channel"
138+
it[avatarUrl] = ""
139+
it[subscribedAt] = 1L
140+
}
141+
}
142+
addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL, watchedAt = 1L)
143+
144+
DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(TEST_USER_ID) }
145+
146+
assertEquals(WATCH_AVATAR_URL, storedAvatar(WATCH_CHANNEL_URL))
147+
assertEquals(WATCH_AVATAR_URL, storedHistoryAvatar())
148+
}
149+
131150
@Test
132151
fun `history reads subscription avatars without waiting for a repair write`() = runTest {
133152
DatabaseFactory.query {

0 commit comments

Comments
 (0)