toki/mattermost-mobile
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[MM-57019] Calls: Live captions support for mobile (#7854)

Browse source 
* mobile support for live captions

* add the 'Live captions turned on' ephemeral notice

* PR comments

* message should be translatable

* run i18n-extract

* call_recording_state -> call_job_state; ccAvailable is now dynamic

* backwards compatibility with pre calls v0.26.0

* correct mobile version in deprecation comments

---------

Co-authored-by: Elias Nahum <nahumhbl@gmail.com>
This commit is contained in:
Christopher Poile 2024-03-20 09:10:05 -04:00 committed by GitHub
parent e48e9f4a88
commit 4e68662899
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 441 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
app/actions/websocket/index.ts
View file

@ -15,10 +15,12 @@ import {openAllUnreadChannels} from '@actions/remote/preference';
import {autoUpdateTimezone} from '@actions/remote/user';
import {loadConfigAndCalls} from '@calls/actions/calls';
import {
handleCallCaption,
handleCallChannelDisabled,
handleCallChannelEnabled,
handleCallEnded,
handleCallHostChanged,
handleCallJobState,
handleCallRecordingState,
handleCallScreenOff,
handleCallScreenOn,
@ -432,15 +434,23 @@ export async function handleEvent(serverUrl: string, msg: WebSocketMessage) {
case WebsocketEvents.CALLS_USER_REACTED:
handleCallUserReacted(serverUrl, msg);
break;
// DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0)
case WebsocketEvents.CALLS_RECORDING_STATE:
handleCallRecordingState(serverUrl, msg);
break;
case WebsocketEvents.CALLS_JOB_STATE:
handleCallJobState(serverUrl, msg);
break;
case WebsocketEvents.CALLS_HOST_CHANGED:
handleCallHostChanged(serverUrl, msg);
break;
case WebsocketEvents.CALLS_USER_DISMISSED_NOTIFICATION:
handleUserDismissedNotification(serverUrl, msg);
break;
case WebsocketEvents.CALLS_CAPTION:
handleCallCaption(serverUrl, msg);
break;
case WebsocketEvents.GROUP_RECEIVED:
handleGroupReceivedEvent(serverUrl, msg);

10
app/constants/calls.ts
View file

@ -24,12 +24,18 @@ const PluginId = 'com.mattermost.calls';
const REACTION_TIMEOUT = 10000;
const REACTION_LIMIT = 20;
const CALL_QUALITY_RESET_MS = toMilliseconds({minutes: 1});
const CAPTION_TIMEOUT = 5000;
export enum MessageBarType {
Microphone,
CallQuality,
}
// The JobTypes from calls plugin's server/public/job.go
const JOB_TYPE_RECORDING = 'recording';
const JOB_TYPE_TRANSCRIBING = 'transcribing';
const JOB_TYPE_CAPTIONING = 'captioning';
export default {
RefreshConfigMillis,
RequiredServer,
@ -39,4 +45,8 @@ export default {
REACTION_LIMIT,
MessageBarType,
CALL_QUALITY_RESET_MS,
CAPTION_TIMEOUT,
JOB_TYPE_RECORDING,
JOB_TYPE_TRANSCRIBING,
JOB_TYPE_CAPTIONING,
};

5
app/constants/websocket.ts
View file

@ -81,9 +81,14 @@ const WebsocketEvents = {
CALLS_USER_RAISE_HAND: `custom_${Calls.PluginId}_user_raise_hand`,
CALLS_USER_UNRAISE_HAND: `custom_${Calls.PluginId}_user_unraise_hand`,
CALLS_USER_REACTED: `custom_${Calls.PluginId}_user_reacted`,
// DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0)
CALLS_RECORDING_STATE: `custom_${Calls.PluginId}_call_recording_state`,
CALLS_JOB_STATE: `custom_${Calls.PluginId}_call_job_state`,
CALLS_HOST_CHANGED: `custom_${Calls.PluginId}_call_host_changed`,
CALLS_USER_DISMISSED_NOTIFICATION: `custom_${Calls.PluginId}_user_dismissed_notification`,
CALLS_CAPTION: `custom_${Calls.PluginId}_caption`,
GROUP_RECEIVED: 'received_group',
GROUP_MEMBER_ADD: 'group_member_add',
GROUP_MEMBER_DELETE: 'group_member_delete',

6
app/products/calls/client/rest.ts
View file

@ -2,7 +2,7 @@
// See LICENSE.txt for license information.
import type {ApiResp, CallsVersion} from '@calls/types/calls';
import type {CallChannelState, CallRecordingState, CallsConfig} from '@mattermost/calls/lib/types';
import type {CallChannelState, CallJobState, CallsConfig} from '@mattermost/calls/lib/types';
import type {RTCIceServer} from 'react-native-webrtc';
export interface ClientCallsMix {
@ -14,8 +14,8 @@ export interface ClientCallsMix {
enableChannelCalls: (channelId: string, enable: boolean) => Promise<CallChannelState>;
endCall: (channelId: string) => Promise<ApiResp>;
genTURNCredentials: () => Promise<RTCIceServer[]>;
startCallRecording: (callId: string) => Promise<ApiResp | CallRecordingState>;
stopCallRecording: (callId: string) => Promise<ApiResp | CallRecordingState>;
startCallRecording: (callId: string) => Promise<ApiResp | CallJobState>;
stopCallRecording: (callId: string) => Promise<ApiResp | CallJobState>;
dismissCall: (channelId: string) => Promise<ApiResp>;
}

119
app/products/calls/components/captions.tsx Normal file
View file

@ -0,0 +1,119 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import React, {useEffect, useState} from 'react';
import {useIntl} from 'react-intl';
import {StyleSheet, Text, View} from 'react-native';
import CompassIcon from '@components/compass_icon';
import FormattedText from '@components/formatted_text';
import {changeOpacity} from '@utils/theme';
import {displayUsername} from '@utils/user';
import type {CallSession, LiveCaptionMobile} from '@calls/types/calls';
const styles = StyleSheet.create({
spacingContainer: {
position: 'relative',
width: '90%',
height: 48,
},
captionContainer: {
display: 'flex',
height: 400,
bottom: -352, // 48-400, to place the bottoms at the same place
gap: 8,
alignItems: 'center',
flexDirection: 'column-reverse',
overflow: 'hidden',
},
caption: {
paddingTop: 1,
paddingRight: 8,
paddingBottom: 3,
paddingLeft: 8,
borderRadius: 4,
backgroundColor: changeOpacity('#000', 0.64),
},
captionNotice: {
display: 'flex',
flexDirection: 'row',
gap: 8,
},
text: {
color: '#FFF',
fontFamily: 'Open Sans',
fontSize: 16,
fontStyle: 'normal',
fontWeight: '400',
lineHeight: 22,
textAlign: 'center',
},
});
type Props = {
captionsDict: Dictionary<LiveCaptionMobile>;
sessionsDict: Dictionary<CallSession>;
teammateNameDisplay: string;
}
const Captions = ({captionsDict, sessionsDict, teammateNameDisplay}: Props) => {
const intl = useIntl();
const [showCCNotice, setShowCCNotice] = useState(true);
useEffect(() => {
const timeoutID = setTimeout(() => {
setShowCCNotice(false);
}, 2000);
return () => clearTimeout(timeoutID);
}, []);
const captionsArr = Object.values(captionsDict).reverse();
if (showCCNotice && captionsArr.length > 0) {
setShowCCNotice(false);
}
if (showCCNotice) {
return (
<View style={styles.spacingContainer}>
<View style={styles.captionContainer}>
<View style={[styles.caption, styles.captionNotice]}>
<CompassIcon
name='closed-caption-outline'
color={'#FFF'}
size={18}
style={{alignSelf: 'center'}}
/>
<FormattedText
id={'mobile.calls_captions_turned_on'}
defaultMessage={'Live captions turned on'}
style={styles.text}
/>
</View>
</View>
</View>
);
}
return (
<View style={styles.spacingContainer}>
<View style={styles.captionContainer}>
{captionsArr.map((cap) => (
<View
key={cap.captionId}
style={styles.caption}
>
<Text
style={styles.text}
numberOfLines={0}
>
{`(${displayUsername(sessionsDict[cap.sessionId]?.userModel, intl.locale, teammateNameDisplay)}) ${cap.text}`}
</Text>
</View>
))}
</View>
</View>
);
};
export default Captions;

33
app/products/calls/connection/websocket_event_handlers.ts
View file

@ -6,10 +6,13 @@ import {DeviceEventEmitter} from 'react-native';
import {fetchUsersByIds} from '@actions/remote/user';
import {
callEnded,
callStarted, getCallsConfig,
callStarted,
getCallsConfig,
receivedCaption,
removeIncomingCall,
setCallScreenOff,
setCallScreenOn,
setCaptioningState,
setChannelEnabled,
setHost,
setRaisedHand,
@ -22,14 +25,18 @@ import {
} from '@calls/state';
import {isMultiSessionSupported} from '@calls/utils';
import {WebsocketEvents} from '@constants';
import Calls from '@constants/calls';
import DatabaseManager from '@database/manager';
import {getCurrentUserId} from '@queries/servers/system';
import type {CallRecordingStateData} from '@calls/types/calls';
import type {
CallHostChangedData,
CallRecordingStateData,
CallJobState,
CallJobStateData,
CallStartData,
EmptyData,
LiveCaptionData,
UserConnectedData,
UserDisconnectedData,
UserDismissedNotification,
@ -155,8 +162,24 @@ export const handleCallUserReacted = (serverUrl: string, msg: WebSocketMessage<U
userReacted(serverUrl, msg.broadcast.channel_id, msg.data);
};
// DEPRECATED in favour of handleCallJobState (since v2.15.0)
export const handleCallRecordingState = (serverUrl: string, msg: WebSocketMessage<CallRecordingStateData>) => {
setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.recState);
const jobState: CallJobState = {
type: Calls.JOB_TYPE_RECORDING,
...msg.data.recState,
};
setRecordingState(serverUrl, msg.broadcast.channel_id, jobState);
};
export const handleCallJobState = (serverUrl: string, msg: WebSocketMessage<CallJobStateData>) => {
switch (msg.data.jobState.type) {
case Calls.JOB_TYPE_RECORDING:
setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.jobState);
break;
case Calls.JOB_TYPE_CAPTIONING:
setCaptioningState(serverUrl, msg.broadcast.channel_id, msg.data.jobState);
break;
}
};
export const handleCallHostChanged = (serverUrl: string, msg: WebSocketMessage<CallHostChangedData>) => {
@ -177,3 +200,7 @@ export const handleUserDismissedNotification = async (serverUrl: string, msg: We
removeIncomingCall(serverUrl, msg.data.callID);
};
export const handleCallCaption = (serverUrl: string, msg: WebSocketMessage<LiveCaptionData>) => {
receivedCaption(serverUrl, msg.data);
};

65
app/products/calls/screens/call_screen/call_screen.tsx
View file

@ -1,5 +1,6 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
/* eslint max-lines: off */
import React, {useCallback, useEffect, useMemo, useState} from 'react';
import {useIntl} from 'react-intl';
@ -29,6 +30,7 @@ import CallAvatar from '@calls/components/call_avatar';
import CallDuration from '@calls/components/call_duration';
import CallNotification from '@calls/components/call_notification';
import CallsBadge, {CallsBadgeType} from '@calls/components/calls_badge';
import Captions from '@calls/components/captions';
import EmojiList from '@calls/components/emoji_list';
import MessageBar from '@calls/components/message_bar';
import ReactionBar from '@calls/components/reaction_bar';
@ -334,6 +336,7 @@ const CallScreen = ({
const [showControlsInLandscape, setShowControlsInLandscape] = useState(false);
const [showReactions, setShowReactions] = useState(false);
const [showCC, setShowCC] = useState(false);
const callsTheme = useMemo(() => makeCallsTheme(theme), [theme]);
const style = getStyleSheet(callsTheme);
const [centerUsers, setCenterUsers] = useState(false);
@ -343,7 +346,7 @@ const CallScreen = ({
const micPermissionsError = !micPermissionsGranted && !currentCall?.micPermissionsErrorDismissed;
const screenShareOn = Boolean(currentCall?.screenOn);
const isLandscape = width > height;
const smallerAvatar = isLandscape || screenShareOn;
const smallerAvatar = isLandscape || screenShareOn || showCC;
const avatarSize = smallerAvatar ? avatarM : avatarL;
const numSessions = Object.keys(sessionsDict).length;
const showIncomingCalls = incomingCalls.incomingCalls.length > 0;
@ -358,6 +361,8 @@ const CallScreen = ({
id: 'mobile.calls_open_channel',
defaultMessage: 'Open Channel',
});
const showCCTitle = intl.formatMessage({id: 'mobile.calls_show_cc', defaultMessage: 'Show live captions'});
const hideCCTitle = intl.formatMessage({id: 'mobile.calls_hide_cc', defaultMessage: 'Hide live captions'});
useEffect(() => {
mergeNavigationOptions('Call', {
@ -424,6 +429,13 @@ const CallScreen = ({
await stopCallRecording(currentCall.serverUrl, currentCall.channelId);
}, [currentCall?.channelId, currentCall?.serverUrl]);
const toggleCC = useCallback(async () => {
Keyboard.dismiss();
await dismissBottomSheet();
setShowCC((prev) => !prev);
}, [setShowCC]);
const switchToThread = useCallback(async () => {
Keyboard.dismiss();
await dismissBottomSheet();
@ -472,6 +484,7 @@ const CallScreen = ({
const waitingForRecording = Boolean(currentCall?.recState?.init_at && !currentCall.recState.start_at && !currentCall.recState.end_at && isHost);
const showStartRecording = isHost && EnableRecordings && !(waitingForRecording || recording);
const showStopRecording = isHost && EnableRecordings && (waitingForRecording || recording);
const ccAvailable = Boolean((currentCall?.capState?.start_at || 0) > (currentCall?.capState?.end_at || 0));
const showOtherActions = useCallback(async () => {
const renderContent = () => {
@ -499,11 +512,22 @@ const CallScreen = ({
onPress={switchToThread}
text={callThreadOptionTitle}
/>
{
ccAvailable &&
<SlideUpPanelItem
leftIcon='closed-caption-outline'
onPress={toggleCC}
text={showCC ? hideCCTitle : showCCTitle}
/>
}
</View>
);
};
const items = isHost && EnableRecordings ? 3 : 2;
let items = isHost && EnableRecordings ? 3 : 2;
if (ccAvailable) {
items++;
}
bottomSheet({
closeButtonId: 'close-other-actions',
renderContent,
@ -512,8 +536,9 @@ const CallScreen = ({
theme,
});
}, [bottom, intl, theme, isHost, EnableRecordings, waitingForRecording, recording, startRecording,
recordOptionTitle, stopRecording, stopRecordingOptionTitle, style, switchToThread, callThreadOptionTitle,
openChannelOptionTitle]);
recordOptionTitle, stopRecording, stopRecordingOptionTitle, style, switchToThread,
callThreadOptionTitle, openChannelOptionTitle, ccAvailable, toggleCC, showCC, hideCCTitle,
showCCTitle]);
const collapse = useCallback(() => {
popTopScreen(componentId);
@ -693,6 +718,13 @@ const CallScreen = ({
{usersList}
{screenShareView}
{isLandscape && header}
{showCC &&
<Captions
captionsDict={currentCall.captions}
sessionsDict={currentCall.sessions}
teammateNameDisplay={teammateNameDisplay}
/>
}
{!isLandscape && currentCall.reactionStream.length > 0 &&
<EmojiList reactionStream={currentCall.reactionStream}/>
}
@ -789,7 +821,7 @@ const CallScreen = ({
style={style.buttonText}
/>
</Pressable>
{!isLandscape && isHost &&
{!isLandscape && (isHost || ccAvailable) &&
<Pressable
style={[style.button, isLandscape && style.buttonLandscape]}
onPress={showOtherActions}
@ -827,7 +859,7 @@ const CallScreen = ({
{mySession.muted ? UnmuteText : MuteText}
</Pressable>
}
{(isLandscape || !isHost) &&
{(isLandscape || (!isHost && !ccAvailable)) &&
<Pressable
style={[style.button, isLandscape && style.buttonLandscape]}
onPress={switchToThread}
@ -846,7 +878,7 @@ const CallScreen = ({
}
{isLandscape && showStartRecording &&
<Pressable
style={[style.button, isLandscape && style.buttonLandscape]}
style={[style.button, style.buttonLandscape]}
onPress={startRecording}
>
<CompassIcon
@ -859,7 +891,7 @@ const CallScreen = ({
}
{isLandscape && showStopRecording &&
<Pressable
style={[style.button, isLandscape && style.buttonLandscape]}
style={[style.button, style.buttonLandscape]}
onPress={stopRecording}
>
<CompassIcon
@ -870,6 +902,23 @@ const CallScreen = ({
<Text style={style.buttonText}>{stopRecordingOptionTitle}</Text>
</Pressable>
}
{isLandscape && ccAvailable &&
<Pressable
style={[style.button, style.buttonLandscape]}
onPress={toggleCC}
>
<CompassIcon
name='closed-caption-outline'
size={32}
style={[style.buttonIcon, style.buttonIconLandscape, showCC && style.buttonOn]}
/>
<FormattedText
id={'mobile.calls_captions'}
defaultMessage={'Captions'}
style={style.buttonText}
/>
</Pressable>
}
</View>
</View>
</View>

84
app/products/calls/state/actions.test.ts
View file

@ -10,6 +10,7 @@ import {
newCurrentCall,
processIncomingCalls,
processMeanOpinionScore,
receivedCaption,
removeIncomingCall,
setAudioDeviceInfo,
setCallQualityAlertDismissed,
@ -60,9 +61,10 @@ import {
type GlobalCallsState,
} from '@calls/types/calls';
import {License} from '@constants';
import Calls from '@constants/calls';
import DatabaseManager from '@database/manager';
import type {CallRecordingState} from '@mattermost/calls/lib/types';
import type {CallJobState, LiveCaptionData} from '@mattermost/calls/lib/types';
jest.mock('@calls/alerts');
@ -1202,7 +1204,8 @@ describe('useCallsState', () => {
myUserId: 'myUserId',
...call1,
};
const recState: CallRecordingState = {
const recState: CallJobState = {
type: Calls.JOB_TYPE_RECORDING,
init_at: 123,
start_at: 231,
end_at: 345,
@ -1407,4 +1410,81 @@ describe('useCallsState', () => {
assert.deepEqual(result.current[1], currentCallStateImIn);
assert.deepEqual(result.current[2], expectedIncomingCalls);
});
it('captions', () => {
const initialCallsState = {
...DefaultCallsState,
serverUrl: 'server1',
myUserId: 'myUserId',
calls: {'channel-1': call1, 'channel-2': call2},
};
const initialCurrentCallState: CurrentCall = {
...DefaultCurrentCall,
serverUrl: 'server1',
myUserId: 'myUserId',
...call1,
};
const caption1user1Data: LiveCaptionData = {
session_id: 'session1',
user_id: 'user-1',
channel_id: 'channel-1',
text: 'caption 1',
};
const caption2user1Data: LiveCaptionData = {
session_id: 'session1',
user_id: 'user-1',
channel_id: 'channel-1',
text: 'caption 2',
};
const caption3user1Data: LiveCaptionData = {
session_id: 'session1',
user_id: 'user-1',
channel_id: 'channel-1',
text: 'caption 3',
};
const caption1user2Data: LiveCaptionData = {
session_id: 'session2',
user_id: 'user-2',
channel_id: 'channel-1',
text: 'caption 1 user 2',
};
const caption2user2Data: LiveCaptionData = {
session_id: 'session2',
user_id: 'user-2',
channel_id: 'channel-1',
text: 'caption 2 user 2',
};
// setup
const {result} = renderHook(() => {
return [useCallsState('server1'), useCurrentCall()];
});
act(() => {
setCallsState('server1', initialCallsState);
setCurrentCall(initialCurrentCallState);
});
assert.deepEqual(result.current[0], initialCallsState);
assert.deepEqual(result.current[1], initialCurrentCallState);
// test sending the first 2 captions for user 1, 1 caption for user 2
act(() => {
receivedCaption('server1', caption1user1Data);
receivedCaption('server1', caption2user1Data);
receivedCaption('server1', caption1user2Data);
});
assert.deepEqual(result.current[0], initialCallsState);
let currentCall = result.current[1] as CurrentCall;
assert.equal(currentCall.captions.session1.text, 'caption 2');
assert.equal(currentCall.captions.session2.text, 'caption 1 user 2');
// test sending the next captions for users 1 and 2
act(() => {
receivedCaption('server1', caption3user1Data);
receivedCaption('server1', caption2user2Data);
});
assert.deepEqual(result.current[0], initialCallsState);
currentCall = result.current[1] as CurrentCall;
assert.equal(currentCall.captions.session1.text, 'caption 3');
assert.equal(currentCall.captions.session2.text, 'caption 2 user 2');
});
});

81
app/products/calls/state/actions.ts
View file

@ -30,6 +30,7 @@ import {
DefaultCall,
DefaultCurrentCall,
type IncomingCallNotification,
type LiveCaptionMobile,
type ReactionStreamEmoji,
} from '@calls/types/calls';
import {Calls, General, Screens} from '@constants';
@ -38,9 +39,10 @@ import {getChannelById} from '@queries/servers/channel';
import {getThreadById} from '@queries/servers/thread';
import {getUserById} from '@queries/servers/user';
import {isDMorGM} from '@utils/channel';
import {generateId} from '@utils/general';
import {logDebug} from '@utils/log';
import type {CallRecordingState, UserReactionData} from '@mattermost/calls/lib/types';
import type {CallJobState, LiveCaptionData, UserReactionData} from '@mattermost/calls/lib/types';
export const setCalls = async (serverUrl: string, myUserId: string, calls: Dictionary<Call>, enabled: Dictionary<boolean>) => {
const channelsWithCalls = Object.keys(calls).reduce(
@ -678,7 +680,7 @@ const userReactionTimeout = (serverUrl: string, channelId: string, reaction: Use
setCurrentCall(nextCurrentCall);
};
export const setRecordingState = (serverUrl: string, channelId: string, recState: CallRecordingState) => {
export const setRecordingState = (serverUrl: string, channelId: string, recState: CallJobState) => {
const callsState = getCallsState(serverUrl);
if (!callsState.calls[channelId]) {
return;
@ -706,6 +708,29 @@ export const setRecordingState = (serverUrl: string, channelId: string, recState
setCurrentCall(nextCurrentCall);
};
export const setCaptioningState = (serverUrl: string, channelId: string, capState: CallJobState) => {
const callsState = getCallsState(serverUrl);
if (!callsState.calls[channelId]) {
return;
}
const nextCall = {...callsState.calls[channelId], capState};
const nextCalls = {...callsState.calls, [channelId]: nextCall};
setCallsState(serverUrl, {...callsState, calls: nextCalls});
// Was it the current call? If so, update that too.
const currentCall = getCurrentCall();
if (!currentCall || currentCall.channelId !== channelId) {
return;
}
const nextCurrentCall = {
...currentCall,
capState,
};
setCurrentCall(nextCurrentCall);
};
export const setHost = (serverUrl: string, channelId: string, hostId: string) => {
const callsState = getCallsState(serverUrl);
if (!callsState.calls[channelId]) {
@ -784,3 +809,55 @@ export const setCallQualityAlertDismissed = () => {
};
setCurrentCall(nextCurrentCall);
};
export const receivedCaption = (serverUrl: string, captionData: LiveCaptionData) => {
const channelId = captionData.channel_id;
// Ignore if we're not in that channel's call.
const currentCall = getCurrentCall();
if (currentCall?.channelId !== channelId) {
return;
}
// Add or replace that user's caption.
const captionId = generateId();
const nextCaptions = {...currentCall.captions};
const newCaption: LiveCaptionMobile = {
captionId,
sessionId: captionData.session_id,
userId: captionData.user_id,
text: captionData.text,
};
nextCaptions[captionData.session_id] = newCaption;
const nextCurrentCall: CurrentCall = {
...currentCall,
captions: nextCaptions,
};
setCurrentCall(nextCurrentCall);
setTimeout(() => {
receivedCaptionTimeout(serverUrl, channelId, newCaption);
}, Calls.CAPTION_TIMEOUT);
};
const receivedCaptionTimeout = (serverUrl: string, channelId: string, caption: LiveCaptionMobile) => {
const currentCall = getCurrentCall();
if (currentCall?.channelId !== channelId) {
return;
}
// Remove the caption only if it hasn't been replaced by a newer one
if (currentCall.captions[caption.sessionId]?.captionId !== caption.captionId) {
return;
}
const nextCaptions = {...currentCall.captions};
delete nextCaptions[caption.sessionId];
const nextCurrentCall: CurrentCall = {
...currentCall,
captions: nextCaptions,
};
setCurrentCall(nextCurrentCall);
};

34
app/products/calls/types/calls.ts
View file

@ -1,7 +1,12 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import type {CallRecordingState, CallsConfig, EmojiData, UserReactionData} from '@mattermost/calls/lib/types';
import type {
CallJobState,
CallsConfig,
EmojiData,
UserReactionData,
} from '@mattermost/calls/lib/types';
import type UserModel from '@typings/database/models/servers/user';
export type GlobalCallsState = {
@ -56,7 +61,8 @@ export type Call = {
screenOn: string;
threadId: string;
ownerId: string;
recState?: CallRecordingState;
recState?: CallJobState;
capState?: CallJobState;
hostId: string;
dismissed: Dictionary<boolean>;
}
@ -94,6 +100,7 @@ export type CurrentCall = Call & {
reactionStream: ReactionStreamEmoji[];
callQualityAlert: boolean;
callQualityAlertDismissed: number;
captions: Dictionary<LiveCaptionMobile>;
}
export const DefaultCurrentCall: CurrentCall = {
@ -110,6 +117,7 @@ export const DefaultCurrentCall: CurrentCall = {
reactionStream: [],
callQualityAlert: false,
callQualityAlertDismissed: 0,
captions: {},
};
export type CallSession = {
@ -158,6 +166,7 @@ export const DefaultCallsConfig: CallsConfigState = {
EnableSimulcast: false,
EnableRinging: false,
EnableTranscriptions: false,
EnableLiveCaptions: false,
};
export type ApiResp = {
@ -205,3 +214,24 @@ export type SelectedSubtitleTrack = {
type: 'system' | 'disabled' | 'title' | 'language' | 'index';
value?: string | number | undefined;
};
export type LiveCaptionMobile = {
captionId: string;
sessionId: string;
userId: string;
text: string;
}
// DEPRECATED in favour of CallJobState since v2.16
export type CallRecordingState = {
init_at: number;
start_at: number;
end_at: number;
err?: string;
error_at?: number;
}
export type CallRecordingStateData = {
recState: CallRecordingState;
callID: string;
}

5
assets/base/i18n/en.json
View file

@ -444,6 +444,8 @@
"mobile.calls_call_ended": "Call ended",
"mobile.calls_call_screen": "Call",
"mobile.calls_call_thread": "Call Thread",
"mobile.calls_captions": "Captions",
"mobile.calls_captions_turned_on": "Live captions turned on",
"mobile.calls_current_call": "Current call",
"mobile.calls_disable": "Disable calls",
"mobile.calls_dismiss": "Dismiss",
@ -458,6 +460,7 @@
"mobile.calls_error_message": "Error: {error}",
"mobile.calls_error_title": "Error",
"mobile.calls_headset": "Headset",
"mobile.calls_hide_cc": "Hide live captions",
"mobile.calls_host": "host",
"mobile.calls_host_rec": "You are recording this meeting. Consider letting everyone know that this meeting is being recorded.",
"mobile.calls_host_rec_error": "Please try to record again. You can also contact your system admin for troubleshooting help.",
@ -509,6 +512,7 @@
"mobile.calls_request_message": "Calls are currently running in test mode and only system admins can start them. Reach out directly to your system admin for assistance",
"mobile.calls_request_title": "Calls is not currently enabled",
"mobile.calls_see_logs": "See server logs",
"mobile.calls_show_cc": "Show live captions",
"mobile.calls_speaker": "Speaker",
"mobile.calls_start_call": "Start Call",
"mobile.calls_start_call_exists": "A call is already ongoing in the channel.",
@ -779,7 +783,6 @@
"notification_settings.mention.reply": "Send reply notifications for",
"notification_settings.mentions": "Mentions",
"notification_settings.mentions_replies": "Mentions and Replies",
"notification_settings.mentions..keywordsDescription": "Other words that trigger a mention",
"notification_settings.mentions.channelWide": "Channel-wide mentions",
"notification_settings.mentions.keywords": "Enter other keywords",
"notification_settings.mentions.keywords_mention": "Keywords that trigger mentions",

1
ios/Mattermost/Info.plist
View file

@ -97,6 +97,7 @@
<string>audio</string>
<string>fetch</string>
<string>remote-notification</string>
<string>voip</string>
</array>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>

13
package-lock.json generated
View file

@ -18,8 +18,8 @@
"@formatjs/intl-pluralrules": "5.2.10",
"@formatjs/intl-relativetimeformat": "11.2.10",
"@gorhom/bottom-sheet": "4.5.1",
"@mattermost/calls": "github:mattermost/calls-common#v0.22.0",
"@mattermost/compass-icons": "0.1.40",
"@mattermost/calls": "github:mattermost/calls-common#954d125d14fd3b46d723bab01d92a0ef3ecca1b8",
"@mattermost/compass-icons": "0.1.43",
"@mattermost/react-native-emm": "1.4.0",
"@mattermost/react-native-network-client": "1.5.0",
"@mattermost/react-native-paste-input": "0.7.0",
@ -3446,7 +3446,8 @@
"node_modules/@mattermost/calls": {
"name": "@calls/common",
"version": "0.14.0",
"resolved": "git+ssh://git@github.com/mattermost/calls-common.git#4a45138c02e3ce45d9e26f81c0f421a136ae0e59"
"resolved": "git+ssh://git@github.com/mattermost/calls-common.git#954d125d14fd3b46d723bab01d92a0ef3ecca1b8",
"integrity": "sha512-dPuvLiWc33WhhTTDqqcurstsh7A3VoGhiAHugQ5rWidvrG1OkzshzTj3jJ0dlCGIUR9EORe+m7M+XeyGgR6KvA=="
},
"node_modules/@mattermost/commonmark": {
"version": "0.30.1-2",
@ -3468,9 +3469,9 @@
}
},
"node_modules/@mattermost/compass-icons": {
"version": "0.1.40",
"resolved": "https://registry.npmjs.org/@mattermost/compass-icons/-/compass-icons-0.1.40.tgz",
"integrity": "sha512-Vz4BeTuwk7bgcKNyOn8F3Q0WsZF8s44bJNawg29pKQc4TLVPurceUhlsqi+B2vVTyCSVyNlFWijg/PWemyCgdQ=="
"version": "0.1.43",
"resolved": "https://registry.npmjs.org/@mattermost/compass-icons/-/compass-icons-0.1.43.tgz",
"integrity": "sha512-vQThJ4SAynnS2u94lQtZ9xANsStpVh8uTpsJascHJOWcavLuL2aDmMLgvg9EAx8Z1qRmTdP6hF5+IU5+9E9+Jg=="
},
"node_modules/@mattermost/react-native-emm": {
"version": "1.4.0",

4
package.json
View file

@ -19,8 +19,8 @@
"@formatjs/intl-pluralrules": "5.2.10",
"@formatjs/intl-relativetimeformat": "11.2.10",
"@gorhom/bottom-sheet": "4.5.1",
"@mattermost/calls": "github:mattermost/calls-common#v0.22.0",
"@mattermost/compass-icons": "0.1.40",
"@mattermost/calls": "github:mattermost/calls-common#954d125d14fd3b46d723bab01d92a0ef3ecca1b8",
"@mattermost/compass-icons": "0.1.43",
"@mattermost/react-native-emm": "1.4.0",
"@mattermost/react-native-network-client": "1.5.0",
"@mattermost/react-native-paste-input": "0.7.0",