diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 02dd77401..a87aeef39 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -146,11 +146,13 @@ "feedback_tab_thank_you": "Thanks, we received your feedback!", "feedback_tab_title": "Feedback", "more_tab_title": "More", + "non_member_tiles": "Show non member tiles", "opt_in_description": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", "preferences_tab_body": "Here you can configure extra options for an improved experience", "preferences_tab_h4": "Preferences", "preferences_tab_show_hand_raised_timer_description": "Show a timer when a participant raises their hand", "preferences_tab_show_hand_raised_timer_label": "Show hand raise duration", + "show_media_keys": "Show media encryption keys", "speaker_device_selection_label": "Speaker" }, "star_rating_input_label_one": "{{count}} stars", diff --git a/src/App.tsx b/src/App.tsx index 8d841dba7..1bc23be83 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -28,6 +28,8 @@ import { Initializer } from "./initializer"; import { MediaDevicesProvider } from "./livekit/MediaDevicesContext"; import { widget } from "./widget"; import { useTheme } from "./useTheme"; +import { nonMemberTiles } from "./settings/settings"; +import { Config } from "./config/Config"; const SentryRoute = Sentry.withSentryRouting(Route); @@ -71,6 +73,13 @@ export const App: FC = ({ history }) => { .catch(logger.error); }); + // Update settings to use the non member tile information from the config if set + useEffect(() => { + if (loaded && Config.get().show_non_member_tiles) { + nonMemberTiles.setValue(true); + } + }); + const errorPage = ; return ( diff --git a/src/Header.tsx b/src/Header.tsx index 69e77935c..c17a0288a 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -117,6 +117,7 @@ interface RoomHeaderInfoProps { avatarUrl: string | null; encrypted: boolean; participantCount: number | null; + nonMemberItemCount: number | null; } export const RoomHeaderInfo: FC = ({ @@ -125,6 +126,7 @@ export const RoomHeaderInfo: FC = ({ avatarUrl, encrypted, participantCount, + nonMemberItemCount, }) => { const { t } = useTranslation(); const size = useMediaQuery("(max-width: 550px)") ? "sm" : "lg"; @@ -157,7 +159,8 @@ export const RoomHeaderInfo: FC = ({ aria-label={t("header_participants_label")} /> - {t("participant_count", { count: participantCount ?? 0 })} + {t("participant_count", { count: participantCount ?? 0 })}{" "} + {(nonMemberItemCount ?? 0) > 0 && <>(+ {nonMemberItemCount})} )} diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 65f04c958..c8f6d3e9c 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -112,6 +112,7 @@ export interface ResolvedConfigOptions extends ConfigOptions { enable_video: boolean; }; app_prompt: boolean; + show_non_member_tiles: boolean; } export const DEFAULT_CONFIG: ResolvedConfigOptions = { @@ -127,4 +128,5 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = { enable_video: true, }, app_prompt: true, + show_non_member_tiles: false, }; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 9492b2f01..ab54f856b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -118,7 +118,7 @@ export const ActiveCall: FC = (props) => { useEffect(() => { if (livekitRoom !== undefined) { const vm = new CallViewModel( - props.rtcSession.room, + props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable, @@ -126,12 +126,7 @@ export const ActiveCall: FC = (props) => { setVm(vm); return (): void => vm.destroy(); } - }, [ - props.rtcSession.room, - livekitRoom, - props.e2eeSystem, - connStateObservable, - ]); + }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]); if (livekitRoom === undefined || vm === null) return null; @@ -194,6 +189,7 @@ export const InCallView: FC = ({ } }, [connState, onLeave]); + const nonMemberItemCount = useObservableEagerState(vm.nonMemberItemCount); const containerRef1 = useRef(null); const [containerRef2, bounds] = useMeasure(); const boundsValid = bounds.height > 0; @@ -633,6 +629,7 @@ export const InCallView: FC = ({ avatarUrl={matrixInfo.roomAvatar} encrypted={matrixInfo.e2eeSystem.kind !== E2eeType.NONE} participantCount={participantCount} + nonMemberItemCount={nonMemberItemCount} /> diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index db702ef8f..2b1b89955 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -27,6 +27,8 @@ import { useSetting, developerSettingsTab as developerSettingsTabSetting, duplicateTiles as duplicateTilesSetting, + nonMemberTiles as nonMemberTilesSetting, + showMediaKeys as showMediaKeysSetting, useOptInAnalytics, } from "./settings"; import { isFirefox } from "../Platform"; @@ -68,6 +70,10 @@ export const SettingsModal: FC = ({ ); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); + const [nonMemberTiles, setNonMemberTiles] = useSetting(nonMemberTilesSetting); + + const [showMediaKeys, setShowMediaKeys] = useSetting(showMediaKeysSetting); + // Generate a `SelectInput` with a list of devices for a given device kind. const generateDeviceSelection = ( devices: MediaDevice, @@ -236,6 +242,34 @@ export const SettingsModal: FC = ({ )} /> + + ): void => { + setNonMemberTiles(event.target.checked); + }, + [setNonMemberTiles], + )} + /> + + + ): void => { + setShowMediaKeys(event.target.checked); + }, + [setShowMediaKeys], + )} + /> + ), }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 109a882b2..f6cf3803f 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -72,6 +72,10 @@ export const developerSettingsTab = new Setting( export const duplicateTiles = new Setting("duplicate-tiles", 0); +export const nonMemberTiles = new Setting("non-member-tiles", false); + +export const showMediaKeys = new Setting("non-member-tiles", false); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index db2833b85..a3455af89 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,12 +18,9 @@ import { RemoteParticipant, Track, } from "livekit-client"; +import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { - Room as MatrixRoom, - RoomMember, - RoomStateEvent, -} from "matrix-js-sdk/src/matrix"; -import { + BehaviorSubject, EMPTY, Observable, Subject, @@ -51,6 +48,10 @@ import { withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; import { ViewModel } from "./ViewModel"; import { @@ -67,7 +68,7 @@ import { } from "./MediaViewModel"; import { accumulate, finalizeValue } from "../utils/observable"; import { ObservableScope } from "./ObservableScope"; -import { duplicateTiles } from "../settings/settings"; +import { duplicateTiles, nonMemberTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; @@ -164,28 +165,40 @@ enum SortingBin { class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; + public participant: BehaviorSubject< + LocalParticipant | RemoteParticipant | undefined + >; + public readonly speaker: Observable; public readonly presenter: Observable; - public constructor( public readonly id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { - this.vm = participant.isLocal - ? new LocalUserMediaViewModel( - id, - member, - participant as LocalParticipant, - encryptionSystem, - ) - : new RemoteUserMediaViewModel( - id, - member, - participant as RemoteParticipant, - encryptionSystem, - ); + this.participant = new BehaviorSubject(participant); + + if (participant && participant.isLocal) { + this.vm = new LocalUserMediaViewModel( + this.id, + member, + this.participant.asObservable() as Observable, + encryptionSystem, + rtcSession, + ); + } else { + this.vm = new RemoteUserMediaViewModel( + id, + member, + this.participant.asObservable() as Observable< + RemoteParticipant | undefined + >, + encryptionSystem, + rtcSession, + ); + } this.speaker = this.vm.speaking.pipe( // Require 1 s of continuous speaking to become a speaker, and 60 s of @@ -195,7 +208,7 @@ class UserMedia { timer(s ? 1000 : 60000), // If the speaking flag resets to its original value during this time, // end the silencing window to stick with that original value - this.vm.speaking.pipe(filter((s1) => s1 !== s)), + this.vm!.speaking.pipe(filter((s1) => s1 !== s)), ), ), startWith(false), @@ -205,13 +218,21 @@ class UserMedia { this.scope.state(), ); - this.presenter = observeParticipantEvents( - participant, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); + this.presenter = this.participant.pipe( + switchMap( + (p) => + (p && + observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled))) ?? + of(false), + ), + this.scope.state(), + ); } public destroy(): void { @@ -222,6 +243,7 @@ class UserMedia { class ScreenShare { public readonly vm: ScreenShareViewModel; + private participant: BehaviorSubject; public constructor( id: string, @@ -229,10 +251,12 @@ class ScreenShare { participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, ) { + this.participant = new BehaviorSubject(participant); + this.vm = new ScreenShareViewModel( id, member, - participant, + this.participant.asObservable(), encryptionSystem, ); } @@ -244,7 +268,7 @@ class ScreenShare { type MediaItem = UserMedia | ScreenShare; -function findMatrixMember( +function findMatrixRoomMember( room: MatrixRoom, id: string, ): RoomMember | undefined { @@ -270,11 +294,11 @@ function findMatrixMember( export class CallViewModel extends ViewModel { public readonly localVideo: Observable = observeTrackReference( - this.livekitRoom.localParticipant, + of(this.livekitRoom.localParticipant), Track.Source.Camera, ).pipe( map((trackRef) => { - const track = trackRef.publication?.track; + const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; }), ); @@ -340,63 +364,160 @@ export class CallViewModel extends ViewModel { }, ); + public readonly nonMemberItemCount = new BehaviorSubject(0); + private readonly mediaItems: Observable = combineLatest([ this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value, - // Also react to changes in the list of members - fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)), + // Also react to changes in the MatrixRTC session list: + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(startWith(null)), + nonMemberTiles.value, ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }, duplicateTiles], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _participantChange, + nonMemberTiles, + ], ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const id = p === localParticipant ? "local" : p.identity; - const member = findMatrixMember(this.matrixRoom, id); - if (member === undefined) - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, + for (const rtcMember of this.matrixRTCSession.memberships) { + const room = this.matrixRTCSession.room; + // WARN! This is not exactly the sender but the user defined in the state key. + // This will be available once we change to the new "member as object" format in the MatrixRTC object. + let mediaId = rtcMember.sender + ":" + rtcMember.deviceId; + let participant = undefined; + if ( + rtcMember.sender === room.client.getUserId()! && + rtcMember.deviceId === room.client.getDeviceId() + ) { + mediaId = "local"; + participant = localParticipant; + } else { + participant = remoteParticipants.find( + (p) => p.identity === mediaId, ); + } - // Create as many tiles for this participant as called for by - // the duplicateTiles option + const member = findMatrixRoomMember(room, mediaId); + if (!member) { + logger.error("Could not find member for media id: ", mediaId); + } for (let i = 0; i < 1 + duplicateTiles; i++) { - const userMediaId = `${id}:${i}`; + const indexedMediaId = `${mediaId}:${i}`; + let prevMedia = prevItems.get(indexedMediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + if (prevMedia.participant.value !== participant) { + // Update the BahviourSubject in the UserMedia. + prevMedia.participant.next(participant); + } + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } + } yield [ - userMediaId, - prevItems.get(userMediaId) ?? + indexedMediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? new UserMedia( - userMediaId, + indexedMediaId, member, - p, + participant, + this.encryptionSystem, + this.matrixRTCSession, + ), + ]; + } + if (participant && participant.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + participant, this.encryptionSystem, ), ]; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, - member, - p, - this.encryptionSystem, - ), - ]; - } } } }.bind(this)(), ); - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); - return newItems; + // Generate non member items (items without a corresponding MatrixRTC member) + // Those items should not be rendered, they are participants in livekit that do not have a corresponding + // matrix rtc members. This cannot be any good: + // - A malicious user impersonates someone + // - Someone injects abusive content + // - The user cannot have encryption keys so it makes no sense to participate + // We can only trust users that have a matrixRTC member event. + // + // This is still available as a debug option. This can be useful + // - If one wants to test scalability using the livekit cli. + // - If an experimental project does not yet do the matrixRTC bits. + // - If someone wants to debug if the LK connection works but matrixRTC room state failed to arrive. + const debugShowNonMember = nonMemberTiles; //Config.get().show_non_member_tiles; + const newNonMemberItems = debugShowNonMember + ? new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const participant of remoteParticipants) { + for (let i = 0; i < 1 + duplicateTiles; i++) { + const maybeNonMemberParticipantId = + participant.identity + ":" + i; + if (!newItems.has(maybeNonMemberParticipantId)) { + const nonMemberId = maybeNonMemberParticipantId; + yield [ + nonMemberId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevItems.get(nonMemberId) ?? + new UserMedia( + nonMemberId, + undefined, + participant, + this.encryptionSystem, + this.matrixRTCSession, + ), + ]; + } + } + } + }.bind(this)(), + ) + : new Map(); + if (newNonMemberItems.size > 0) { + logger.debug("Added NonMember items: ", newNonMemberItems); + } + const newNonMemberItemCount = + newNonMemberItems.size / (1 + duplicateTiles); + if (this.nonMemberItemCount.value !== newNonMemberItemCount) + this.nonMemberItemCount.next(newNonMemberItemCount); + + const combinedNew = new Map([ + ...newNonMemberItems.entries(), + ...newItems.entries(), + ]); + + for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy(); + return combinedNew; }, new Map(), ), @@ -432,42 +553,43 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly spotlightSpeaker: Observable = - this.userMedia.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking.pipe(map((s) => [m, s] as const)), - ), + private readonly spotlightSpeaker: Observable< + UserMediaViewModel | undefined + > = this.userMedia.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking.pipe(map((s) => [m, s] as const)), ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)![0]); - }, - null, - ), - map((speaker) => speaker.vm), - this.scope.state(), - throttleTime(1600, undefined, { leading: true, trailing: true }), - ); + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm), + this.scope.state(), + throttleTime(1600, undefined, { leading: true, trailing: true }), + ); private readonly grid: Observable = this.userMedia.pipe( switchMap((mediaItems) => { @@ -510,20 +632,29 @@ export class CallViewModel extends ViewModel { > = this.screenShares.pipe( map((screenShares) => screenShares.length > 0 - ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) + ? ([ + of(screenShares.map((m) => m.vm)), + this.spotlightSpeaker.pipe( + map((speaker) => (speaker && speaker) ?? null), + ), + ] as const) : ([ - this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), + this.spotlightSpeaker.pipe( + map((speaker) => (speaker && [speaker]) ?? []), + ), this.spotlightSpeaker.pipe( switchMap((speaker) => - speaker.local - ? of(null) - : this.localUserMedia.pipe( - switchMap((vm) => - vm.alwaysShow.pipe( - map((alwaysShow) => (alwaysShow ? vm : null)), + speaker + ? speaker.local + ? of(null) + : this.localUserMedia.pipe( + switchMap((vm) => + vm.alwaysShow.pipe( + map((alwaysShow) => (alwaysShow ? vm : null)), + ), ), - ), - ), + ) + : of(null), ), ), ] as const), @@ -843,7 +974,7 @@ export class CallViewModel extends ViewModel { public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encryptionSystem: EncryptionSystem, private readonly connectionState: Observable, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 51a821af1..f3257243f 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -37,6 +37,10 @@ import { switchMap, } from "rxjs"; import { useEffect } from "react"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; import { ViewModel } from "./ViewModel"; import { useReactiveState } from "../useReactiveState"; @@ -68,33 +72,50 @@ export function useDisplayName(vm: MediaViewModel): string { } export function observeTrackReference( - participant: Participant, + participant: Observable, source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), +): Observable { + const obs = participant.pipe( + switchMap((p) => { + if (p) { + return observeParticipantMedia(p).pipe( + map(() => ({ + participant: p, + publication: p.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + ); + } else { + return of(undefined); + } + }), ); + return obs; } abstract class BaseMediaViewModel extends ViewModel { /** * Whether the media belongs to the local user. */ - public readonly local = this.participant.isLocal; + public readonly local = this.participant.pipe( + // We can assume, that the user is not local if the participant is undefined + // We assume the local LK participant will always be available. + map((p) => p?.isLocal ?? false), + ); /** * The LiveKit video track for this media. */ - public readonly video: Observable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ public readonly unencryptedWarning: Observable; + public readonly isRTCParticipantAvailable = this.participant.pipe( + map((p) => !!p), + ); + public constructor( /** * An opaque identifier for this media. @@ -106,7 +127,12 @@ abstract class BaseMediaViewModel extends ViewModel { // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal public readonly member: RoomMember | undefined, - protected readonly participant: LocalParticipant | RemoteParticipant, + // We dont necassarly have a participant if a user connects via MatrixRTC but not (not yet) through + // livekit. + protected readonly participant: Observable< + LocalParticipant | RemoteParticipant | undefined + >, + encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, @@ -122,8 +148,8 @@ abstract class BaseMediaViewModel extends ViewModel { [audio, this.video], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), + (a?.publication?.isEncrypted === false || + v?.publication?.isEncrypted === false), ).pipe(this.scope.state()); } } @@ -143,12 +169,20 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe( - map((p) => p.isSpeaking), - this.scope.state(), + public readonly speaking = this.participant.pipe( + switchMap((p) => { + if (p) { + return observeParticipantEvents( + p, + ParticipantEvent.IsSpeakingChanged, + ).pipe( + map((p) => p.isSpeaking), + this.scope.state(), + ); + } else { + return of(false); + } + }), ); /** @@ -166,11 +200,16 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo: Observable = this._cropVideo; + public readonly keys = new BehaviorSubject( + [] as { index: number; key: Uint8Array }[], + ); + public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { super( id, @@ -181,12 +220,32 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { Track.Source.Camera, ); - const media = observeParticipantMedia(participant).pipe(this.scope.state()); + combineLatest([ + participant, + fromEvent(rtcSession, MatrixRTCSessionEvent.EncryptionKeyChanged).pipe( + startWith(null), + ), + ]).subscribe(([par, ev]) => { + for (const participantKeys of rtcSession.getEncryptionKeys()) { + if (participantKeys[0] === par?.identity) { + this.keys.next( + Array.from(participantKeys[1].entries()).map(([i, k]) => { + return { index: i, key: k }; + }), + ); + } + } + }); + + const media = participant.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + this.scope.state(), + ); this.audioEnabled = media.pipe( - map((m) => m.microphoneTrack?.isMuted === false), + map((m) => m?.microphoneTrack?.isMuted === false), ); this.videoEnabled = media.pipe( - map((m) => m.cameraTrack?.isMuted === false), + map((m) => m?.cameraTrack?.isMuted === false), ); } @@ -204,7 +263,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { */ public readonly mirror = this.video.pipe( switchMap((v) => { - const track = v.publication?.track; + const track = v?.publication?.track; if (!(track instanceof LocalTrack)) return of(false); // Watch for track restarts, because they indicate a camera switch return fromEvent(track, TrackEvent.Restarted).pipe( @@ -226,10 +285,11 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, rtcSession); } } @@ -286,17 +346,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, rtcSession); // Sync the local volume with LiveKit - this.localVolume - .pipe(this.scope.bind()) - .subscribe((volume) => - (this.participant as RemoteParticipant).setVolume(volume), - ); + combineLatest([ + participant, + this.localVolume.pipe(this.scope.bind()), + ]).subscribe(([p, volume]) => p && p.setVolume(volume)); } public toggleLocallyMuted(): void { @@ -319,7 +379,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, ) { super( diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3675e9a7c..980cb4f26 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -84,6 +84,10 @@ const UserMediaTile = forwardRef( const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); const cropVideo = useObservableEagerState(vm.cropVideo); + const keys = useObservableEagerState(vm.keys); + const isRTCParticipantAvailable = useObservableEagerState( + vm.isRTCParticipantAvailable, + ); const onSelectFitContain = useCallback( (e: Event) => { e.preventDefault(); @@ -118,6 +122,7 @@ const UserMediaTile = forwardRef( ref={ref} video={video} member={vm.member} + keys={keys} unencryptedWarning={unencryptedWarning} videoEnabled={videoEnabled && showVideo} videoFit={cropVideo ? "cover" : "contain"} @@ -134,7 +139,10 @@ const UserMediaTile = forwardRef( className={styles.muteIcon} /> } - displayName={displayName} + displayName={ + displayName + + (isRTCParticipantAvailable ? "" : " missing Livekit Participant...") + } primaryButton={ { e.preventDefault(); @@ -248,6 +257,9 @@ const RemoteUserMediaTile = forwardRef< mirror={false} menuStart={ <> + {/* {isRTCParticipantAvailable + ? "is available" + : "Loading RTC participant"} */} { className?: string; style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder; + video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; member: RoomMember | undefined; + keys: { index: number; key: Uint8Array }[]; videoEnabled: boolean; unencryptedWarning: boolean; nameTagLeadingIcon?: ReactNode; @@ -48,6 +53,7 @@ export const MediaView = forwardRef( videoFit, mirror, member, + keys, videoEnabled, unencryptedWarning, nameTagLeadingIcon, @@ -60,6 +66,7 @@ export const MediaView = forwardRef( ) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); + const [showMediaKeys] = useSetting(showMediaKeysSettings); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -83,7 +90,7 @@ export const MediaView = forwardRef( src={member?.getMxcAvatarUrl()} className={styles.avatar} /> - {video.publication !== undefined && ( + {video?.publication !== undefined && ( ( minature={avatarSize < 96} showTimer={handRaiseTimerVisible} /> + {keys && showMediaKeys && }
{nameTagLeadingIcon} {displayName} + {unencryptedWarning && ( ( ); }, ); +interface MediaKeyListProps { + keys: { + index: number; + key: Uint8Array; + }[]; +} +export const MediaKeyList: FC = ({ keys }) => { + return ( +
+ {keys.map(({ index, key }) => ( +
+ + index:{index} + + + key:{key ? encodeUnpaddedBase64(key) : "unavailable"} + +
+ ))} +
+ ); +}; MediaView.displayName = "MediaView";