feat(calls): 複数の画面共有と共有音声ごとの音量調整に対応

docs/calls-protocol.md

@@ -72,3 +72,5 @@ Room owners can appoint or dismiss an active non-host participant through `calls
 Only the current host can transfer an open room to an active non-host participant through `calls/rooms/transfer-host` (`roomId`, `participantId`, `expectedRevision`). Ownership and both participant roles change in one transaction: the target becomes host and the previous host remains a speaker, so their later exit does not end the room. The target's moderator assignment and speaker request are cleared. Existing participants retain access to non-public rooms; follower visibility subsequently follows the new owner. A target who already owns an active personal room is rejected with `CALLS_ACTIVE_ATTACHMENT`. A `participant` event refreshes the room and participant snapshot, and the host timeout is renewed.
 
 Host transfer checks the target’s current Calls participation permission. Its moderation log records the original actor, target and committed revision. Repeating the same request with the original `expectedRevision` can resend notifications after a committed transfer without changing ownership again, provided the recorded target is still host. Clients refresh the committed state on an ambiguous failure and retry that notification once. The previous host receives a role update, and the new host’s participant update refreshes the room snapshot for viewers.
+
+A participant may publish multiple screen videos. To attach screen audio, publish `mediaSource: screen` with `screenPublicationId` referencing their current screen video publication. This publishes an audio track, and reconcile includes its `screenPublicationId` so receivers can control each share’s volume separately from microphone volume. Screen capture requests audio through the browser picker; only audio explicitly supplied by the browser is published. Stopping a share closes both of its tracks.

locales/ja-JP.yml

@@ -2747,6 +2747,8 @@ _calls:
   stopCamera: "カメラをオフ"
   screenSharing: "画面共有"
   startScreenSharing: "画面を共有"
+  addScreenSharing: "画面共有を追加"
+  screenAudioVolume: "画面共有の音量"
   stopScreenSharing: "共有を停止"
   videoPermissionDenied: "カメラへのアクセスを許可してください。"
   cameraNotFound: "カメラが見つかりません。"

packages/backend/src/core/calls/CallsMediaBindingService.ts

@@ -20,6 +20,7 @@ export type CallsPublicationBinding = {
 	providerMid: string | null;
 	mediaKind: 'audio' | 'video';
 	mediaSource?: 'microphone' | 'camera' | 'screen';
+	screenPublicationId?: string;
 	createdAt: string;
 };
 

packages/backend/src/core/calls/CallsMediaService.ts

@@ -97,6 +97,7 @@ export class CallsMediaService {
 		generation: number;
 		mid: string;
 		mediaSource?: 'microphone' | 'camera' | 'screen';
+		screenPublicationId?: string;
 		sessionDescription: CloudflareRealtimeSessionDescription;
 	}): Promise<{ publicationId: string; negotiation: CloudflareRealtimeTracksResponse }> {
 		const participant = await this.authorizeParticipant(user, input.roomId);
@@ -105,7 +106,11 @@ export class CallsMediaService {
 		if (connection.sessionId == null) throw new CallsMediaAccessError();
 		const mediaSource = input.mediaSource ?? 'microphone';
 		await this.roomService.assertCanPublish(user, mediaSource);
-		const mediaKind = mediaSource === 'microphone' ? 'audio' : 'video';
+		if (input.screenPublicationId != null) {
+			const screen = await this.bindingService.getPublication(input.screenPublicationId);
+			if (mediaSource !== 'screen' || screen.roomId !== input.roomId || screen.participantId !== participant.id || screen.connectionId !== input.connectionId || screen.generation !== input.generation || screen.mediaKind !== 'video' || screen.mediaSource !== 'screen') throw new CallsMediaAccessError();
+		}
+		const mediaKind = mediaSource === 'microphone' || input.screenPublicationId != null ? 'audio' : 'video';
 		const trackName = `${mediaSource}-${participant.id}-${input.generation}-${input.mid}`;
 		await this.quotaService.reserveTrack(connection.applicationId, trackName);
 		let providerMid: string | null = null;
@@ -128,7 +133,7 @@ export class CallsMediaService {
 				providerSessionId: connection.sessionId,
 				providerTrackName: track?.trackName ?? trackName,
 				providerMid: track?.mid ?? input.mid,
-				mediaKind, mediaSource,
+				mediaKind, mediaSource, screenPublicationId: input.screenPublicationId,
 			});
 			await this.liveConnectionService.assertCurrent(participant.id, input.connectionId, input.generation);
 			const room = await this.roomService.getRoom(input.roomId);
@@ -206,11 +211,11 @@ export class CallsMediaService {
 		return response;
 	}
 
-	public async reconcile(user: MiUser, roomId: string): Promise<{ roomRevision: number; publications: Array<{ id: string; participantId: string; mediaKind: 'audio' | 'video'; mediaSource: 'microphone' | 'camera' | 'screen' }> }> {
+	public async reconcile(user: MiUser, roomId: string): Promise<{ roomRevision: number; publications: Array<{ id: string; participantId: string; mediaKind: 'audio' | 'video'; mediaSource: 'microphone' | 'camera' | 'screen'; screenPublicationId?: string }> }> {
 		const snapshot = await this.roomService.snapshot(user, roomId);
 		const activeSpeakers = new Set(snapshot.participants.filter(p => p.role !== 'listener').map(p => p.id));
 		const publications = (await this.bindingService.listRoomPublications(roomId)).filter(binding => activeSpeakers.has(binding.participantId));
-		return { roomRevision: snapshot.room.revision, publications: publications.map(binding => ({ id: binding.id, participantId: binding.participantId, mediaKind: binding.mediaKind, mediaSource: binding.mediaSource ?? 'microphone' })) };
+		return { roomRevision: snapshot.room.revision, publications: publications.map(binding => ({ id: binding.id, participantId: binding.participantId, mediaKind: binding.mediaKind, mediaSource: binding.mediaSource ?? 'microphone', ...(binding.screenPublicationId == null ? {} : { screenPublicationId: binding.screenPublicationId }) })) };
 	}
 
 	public async heartbeat(user: MiUser, roomId: string, connectionId: string, generation: number): Promise<void> {

packages/backend/src/server/api/endpoints/calls/media/publish.ts

@@ -10,7 +10,7 @@ import { CallsOperationGuardService } from '@/core/calls/CallsOperationGuardServ
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { callsMediaApiError, callsMediaErrors, executeCallsMediaOperation, negotiationResponseSchema, operationIdParam, sanitizeNegotiation, sessionDescriptionParam, verifyCallsMediaCredential } from '../_media-shared.js';
 export const meta = { tags: ['calls'], stability: 'experimental', requireCredential: true, prohibitMoved: true, kind: 'write:calls', limit: { key: 'calls-media', duration: 60 * 1000, max: 120 }, errors: callsMediaErrors, res: { type: 'object', optional: false, nullable: false, properties: { publicationId: { type: 'string', format: 'id', optional: false, nullable: false }, negotiation: negotiationResponseSchema } } } as const;
-export const paramDef = { type: 'object', properties: { roomId: { type: 'string', format: 'misskey:id' }, participantId: { type: 'string', format: 'misskey:id' }, connectionId: { type: 'string', minLength: 8, maxLength: 128 }, generation: { type: 'integer', minimum: 1 }, operationId: operationIdParam, mediaCredential: { type: 'string', minLength: 32, maxLength: 4096 }, mediaSource: { type: 'string', enum: ['microphone', 'camera', 'screen'], default: 'microphone' }, mid: { type: 'string', minLength: 1, maxLength: 64 }, sessionDescription: sessionDescriptionParam }, required: ['roomId', 'participantId', 'connectionId', 'generation', 'operationId', 'mediaCredential', 'mid', 'sessionDescription'] } as const;
+export const paramDef = { type: 'object', properties: { roomId: { type: 'string', format: 'misskey:id' }, participantId: { type: 'string', format: 'misskey:id' }, connectionId: { type: 'string', minLength: 8, maxLength: 128 }, generation: { type: 'integer', minimum: 1 }, operationId: operationIdParam, mediaCredential: { type: 'string', minLength: 32, maxLength: 4096 }, mediaSource: { type: 'string', enum: ['microphone', 'camera', 'screen'], default: 'microphone' }, screenPublicationId: { type: 'string', format: 'misskey:id' }, mid: { type: 'string', minLength: 1, maxLength: 64 }, sessionDescription: sessionDescriptionParam }, required: ['roomId', 'participantId', 'connectionId', 'generation', 'operationId', 'mediaCredential', 'mid', 'sessionDescription'] } as const;
 @Injectable()
 export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
 	constructor(service: CallsMediaService, credentials: CallsMediaCredentialService, guard: CallsOperationGuardService) { super(meta, paramDef, async (ps, me, token) => { try { verifyCallsMediaCredential(credentials, ps.mediaCredential, me, token, ps, true); return await executeCallsMediaOperation(guard, me, token, ps, 'publish', async () => { const result = await service.publish(me, ps); return { publicationId: result.publicationId, negotiation: sanitizeNegotiation(result.negotiation) }; }); } catch (error) { callsMediaApiError(error); } }); }

packages/backend/src/server/api/endpoints/calls/media/reconcile.ts

@@ -9,7 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
 import { callsMediaApiError, callsMediaErrors } from '../_media-shared.js';
 export const meta = { tags: ['calls'], stability: 'experimental', requireCredential: true, kind: 'read:calls', errors: callsMediaErrors, res: { type: 'object', optional: false, nullable: false, properties: {
 	roomRevision: { type: 'integer', optional: false, nullable: false },
-	publications: { type: 'array', optional: false, nullable: false, items: { type: 'object', properties: { id: { type: 'string', format: 'id', optional: false, nullable: false }, participantId: { type: 'string', format: 'id', optional: false, nullable: false }, mediaSource: { type: 'string', enum: ['microphone', 'camera', 'screen'], optional: false, nullable: false }, mediaKind: { type: 'string', enum: ['audio', 'video'], optional: false, nullable: false } } } },
+	publications: { type: 'array', optional: false, nullable: false, items: { type: 'object', properties: { id: { type: 'string', format: 'id', optional: false, nullable: false }, participantId: { type: 'string', format: 'id', optional: false, nullable: false }, mediaSource: { type: 'string', enum: ['microphone', 'camera', 'screen'], optional: false, nullable: false }, screenPublicationId: { type: 'string', format: 'id', optional: true, nullable: false }, mediaKind: { type: 'string', enum: ['audio', 'video'], optional: false, nullable: false } } } },
 } } } as const;
 export const paramDef = { type: 'object', properties: { roomId: { type: 'string', format: 'misskey:id' } }, required: ['roomId'] } as const;
 @Injectable()

packages/backend/test/unit/core/calls/CallsMediaService.ts

@@ -137,6 +137,23 @@ describe('CallsMediaService authorization boundaries', () => {
 		expect(fixture.quota.reserveTrack).toHaveBeenCalledWith('app-a', `${mediaSource}-participant-a-2-1`);
 	});
 
+	test('publishes screen audio linked to the current owned screen video', async () => {
+		const fixture = createFixture();
+		fixture.bindings.getPublication.mockResolvedValue({ id: 'screen-a', roomId: room.id, participantId: 'participant-a', connectionId: 'connection-a', generation: 2, mediaKind: 'video', mediaSource: 'screen' });
+		fixture.provider.addTracks.mockResolvedValue({ tracks: [{ mid: '2' }] });
+		fixture.bindings.createPublication.mockResolvedValue({ id: 'screen-audio' });
+		await fixture.service.publish(user, { roomId: room.id, connectionId: 'connection-a', generation: 2, mid: '2', mediaSource: 'screen', screenPublicationId: 'screen-a', sessionDescription: { type: 'offer', sdp: 'sdp' } });
+		expect(fixture.provider.addTracks).toHaveBeenCalledWith('session-a', [expect.objectContaining({ kind: 'audio' })], expect.anything());
+		expect(fixture.bindings.createPublication).toHaveBeenCalledWith(expect.objectContaining({ mediaKind: 'audio', mediaSource: 'screen', screenPublicationId: 'screen-a' }));
+	});
+
+	test.each([{ participantId: 'participant-b' }, { roomId: 'room-b' }, { generation: 1 }, { mediaSource: 'camera' }])('rejects attaching screen audio to a foreign or stale publication: %s', async invalid => {
+		const fixture = createFixture();
+		fixture.bindings.getPublication.mockResolvedValue({ roomId: room.id, participantId: 'participant-a', connectionId: 'connection-a', generation: 2, mediaKind: 'video', mediaSource: 'screen', ...invalid });
+		await expect(fixture.service.publish(user, { roomId: room.id, connectionId: 'connection-a', generation: 2, mid: '2', mediaSource: 'screen', screenPublicationId: 'screen-a', sessionDescription: { type: 'offer', sdp: 'sdp' } })).rejects.toBeInstanceOf(CallsMediaAccessError);
+		expect(fixture.provider.addTracks).not.toHaveBeenCalled();
+	});
+
 	test('maps subscription mids to public IDs even when provider results are reordered', async () => {
 		const fixture = createFixture('listener');
 		const publications = [

packages/frontend/src/components/MkCallsRoomWindow.vue

@@ -36,7 +36,7 @@
 				<div :class="$style.callLayout">
 					<section ref="stage" :class="$style.stage" :aria-label="i18n.ts._calls.title">
 						<div ref="videoGrid" :class="$style.videoGrid" :style="videoGridStyle">
-							<CallsVideo v-for="video in roomVideos" :key="video.id" :stream="video.stream" screenWindow :screenWindowActive="session.screenWindows.has(video.stream)" :label="videoLabel(video.participantId)" :speaking="speakingParticipantIds.has(video.participantId)" :focused="focusedVideoId === video.id" :class="focusedVideoId === video.id && $style.focusedVideo" @contextmenu.capture.stop.prevent="openParticipantMenu(participants.find(participant => participant.id === video.participantId), $event)" @select="focusVideo(video.id)" @screenWindow="showScreenWindow(video.stream, video.participantId)"/>
+							<CallsVideo v-for="video in roomVideos" :key="video.id" :stream="video.stream" :audioVolume="!video.local && session.screenAudioIds.value.has(video.id) ? session.getScreenVolume(video.id) : undefined" screenWindow :screenWindowActive="session.screenWindows.has(video.stream)" :label="videoLabel(video.participantId)" :speaking="speakingParticipantIds.has(video.participantId)" :focused="focusedVideoId === video.id" :class="focusedVideoId === video.id && $style.focusedVideo" @contextmenu.capture.stop.prevent="openParticipantMenu(participants.find(participant => participant.id === video.participantId), $event)" @volume="session.setScreenVolume(video.id, $event)" @select="focusVideo(video.id)" @screenWindow="showScreenWindow(video.stream, video.participantId)"/>
 							<div v-for="participant in audioOnlySpeakers" :key="participant.id" :class="[$style.voiceTile, speakingParticipantIds.has(participant.id) && $style.speaking]" @contextmenu.capture.stop.prevent="openParticipantMenu(participant, $event)">
 								<MkAvatar v-if="participantUser(participant.userId) != null" :user="participantUser(participant.userId)!" :class="$style.stageAvatar"/>
 								<i v-else class="ti ti-user" :class="$style.stageAvatarPlaceholder"></i>

packages/frontend/src/components/MkCallsScreenWindow.vue

@@ -6,13 +6,17 @@ SPDX-License-Identifier: AGPL-3.0-only
 <template>
 <MkWindow :initialWidth="480" :initialHeight="320" canResize front @closed="emit('closed')">
 	<template #header><i class="ti ti-app-window"></i> {{ label }}</template>
-	<video ref="video" autoplay muted playsinline :aria-label="label" :class="$style.video" @canplay="play"></video>
+	<div :class="$style.content">
+		<video ref="video" autoplay muted playsinline :aria-label="label" :class="$style.video" @canplay="play"></video>
+		<label v-if="screenVideo != null && session.screenAudioIds.value.has(screenVideo.id)" :class="$style.volume"><i class="ti ti-volume"></i><input type="range" min="0" max="100" step="1" :value="session.getScreenVolume(screenVideo.id)" :aria-label="i18n.ts._calls.screenAudioVolume" @input="session.setScreenVolume(screenVideo.id, ($event.target as HTMLInputElement).valueAsNumber)"></label>
+	</div>
 </MkWindow>
 </template>
 
 <script setup lang="ts">
-import { onUnmounted, shallowRef, watch } from 'vue';
+import { computed, onUnmounted, shallowRef, watch } from 'vue';
 import MkWindow from '@/components/MkWindow.vue';
+import { useCallsSession } from '@/utility/calls-session.js';
 import * as os from '@/os.js';
 import { i18n } from '@/i18n.js';
 
@@ -22,6 +26,8 @@ const props = defineProps<{
 }>();
 
 const emit = defineEmits<{ (ev: 'closed'): void }>();
+const session = useCallsSession();
+const screenVideo = computed(() => session.videos.value.find(item => !item.local && item.stream === props.stream));
 const video = shallowRef<HTMLVideoElement | null>(null);
 
 function play(): void {
@@ -45,5 +51,8 @@ onUnmounted(() => {
 </script>
 
 <style lang="scss" module>
+.content { position: relative; height: 100%; }
+.volume { position: absolute; top: 8px; right: 8px; display: flex; align-items: center; gap: 6px; padding: 6px 8px; border-radius: 8px; background: var(--MI_THEME-panel); }
+.volume > input { width: 100px; accent-color: var(--MI_THEME-accent); }
 .video { display: block; width: 100%; height: 100%; object-fit: contain; background: var(--MI_THEME-bg); }
 </style>

packages/frontend/src/components/MkCallsVideo.stories.impl.ts

@@ -32,3 +32,5 @@ export const Default = {
 export const Speaking = { ...Default, args: { ...Default.args, speaking: true } } satisfies StoryObj<typeof MkCallsVideo>;
 
 export const Screen = { ...Default, args: { ...Default.args, screenWindow: true } } satisfies StoryObj<typeof MkCallsVideo>;
+
+export const ScreenWithAudio = { ...Screen, args: { ...Screen.args, audioVolume: 75 } } satisfies StoryObj<typeof MkCallsVideo>;

packages/frontend/src/components/MkCallsVideo.vue

@@ -9,6 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 	<button type="button" class="_button" :class="$style.select" :aria-label="focused ? i18n.ts._calls.showVideoGrid : i18n.ts._calls.focusVideo" :aria-pressed="focused" @click="emit('select')"></button>
 	<button v-if="needsPlayback" type="button" class="_button" :class="$style.play" :aria-label="i18n.ts._calls.resumeVideo" @click="startPlayback"><i class="ti ti-player-play"></i> {{ i18n.ts._calls.resumeVideo }}</button>
 	<button v-if="screenWindow" type="button" class="_button" :class="$style.windowButton" :aria-label="i18n.ts.openInWindow" :title="i18n.ts.openInWindow" :aria-pressed="screenWindowActive" @click="emit('screenWindow')"><i class="ti ti-app-window"></i></button>
+	<label v-if="audioVolume != null" :class="$style.volume" @click.stop @contextmenu.stop><i class="ti ti-volume"></i><input type="range" min="0" max="100" step="1" :value="audioVolume" :aria-label="`${i18n.ts._calls.screenAudioVolume}: ${label}`" @input="emit('volume', ($event.target as HTMLInputElement).valueAsNumber)"></label>
 	<figcaption :class="$style.caption"><span>{{ label }}</span></figcaption>
 </figure>
 </template>
@@ -24,8 +25,9 @@ const props = defineProps<{
 	focused?: boolean;
 	screenWindow?: boolean;
 	screenWindowActive?: boolean;
+	audioVolume?: number;
 }>();
-const emit = defineEmits<{ (ev: 'select' | 'screenWindow'): void }>();
+const emit = defineEmits<{ (ev: 'select' | 'screenWindow'): void; (ev: 'volume', value: number): void }>();
 const video = shallowRef<HTMLVideoElement | null>(null);
 const needsPlayback = shallowRef(false);
 
@@ -99,6 +101,9 @@ watch([video, () => props.stream], ([element, stream]) => {
 	white-space: nowrap;
 }
 
+.volume { position: absolute; top: 8px; right: 8px; display: flex; align-items: center; gap: 6px; padding: 6px 8px; border-radius: 8px; background: var(--MI_THEME-panel); }
+.volume > input { width: 100px; accent-color: var(--MI_THEME-accent); }
+
 .windowButton { position: absolute; bottom: 8px; right: 8px; display: grid; place-items: center; width: 40px; height: 40px; border-radius: 8px; background: var(--MI_THEME-panel); font-size: 20px; opacity: 0; transition: opacity 0.18s ease; }
 .play { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; gap: 8px; background: var(--MI_THEME-panel); }
 @media (pointer: coarse) { .caption, .windowButton { opacity: 1; } }