From 4e686628990fded352f9f62a9976b10f5f1fb850 Mon Sep 17 00:00:00 2001 From: Christopher Poile Date: Wed, 20 Mar 2024 09:10:05 -0400 Subject: [PATCH] [MM-57019] Calls: Live captions support for mobile (#7854) * mobile support for live captions * add the 'Live captions turned on' ephemeral notice * PR comments * message should be translatable * run i18n-extract * call_recording_state -> call_job_state; ccAvailable is now dynamic * backwards compatibility with pre calls v0.26.0 * correct mobile version in deprecation comments --------- Co-authored-by: Elias Nahum --- app/actions/websocket/index.ts | 10 ++ app/constants/calls.ts | 10 ++ app/constants/websocket.ts | 5 + app/products/calls/client/rest.ts | 6 +- app/products/calls/components/captions.tsx | 119 ++++++++++++++++++ .../connection/websocket_event_handlers.ts | 33 ++++- .../calls/screens/call_screen/call_screen.tsx | 65 ++++++++-- app/products/calls/state/actions.test.ts | 84 ++++++++++++- app/products/calls/state/actions.ts | 81 +++++++++++- app/products/calls/types/calls.ts | 34 ++++- assets/base/i18n/en.json | 5 +- ios/Mattermost/Info.plist | 1 + package-lock.json | 13 +- package.json | 4 +- 14 files changed, 441 insertions(+), 29 deletions(-) create mode 100644 app/products/calls/components/captions.tsx diff --git a/app/actions/websocket/index.ts b/app/actions/websocket/index.ts index 2a8f1f297..779df9f28 100644 --- a/app/actions/websocket/index.ts +++ b/app/actions/websocket/index.ts @@ -15,10 +15,12 @@ import {openAllUnreadChannels} from '@actions/remote/preference'; import {autoUpdateTimezone} from '@actions/remote/user'; import {loadConfigAndCalls} from '@calls/actions/calls'; import { + handleCallCaption, handleCallChannelDisabled, handleCallChannelEnabled, handleCallEnded, handleCallHostChanged, + handleCallJobState, handleCallRecordingState, handleCallScreenOff, handleCallScreenOn, @@ -432,15 +434,23 @@ export async function handleEvent(serverUrl: string, msg: WebSocketMessage) { case WebsocketEvents.CALLS_USER_REACTED: handleCallUserReacted(serverUrl, msg); break; + + // DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0) case WebsocketEvents.CALLS_RECORDING_STATE: handleCallRecordingState(serverUrl, msg); break; + case WebsocketEvents.CALLS_JOB_STATE: + handleCallJobState(serverUrl, msg); + break; case WebsocketEvents.CALLS_HOST_CHANGED: handleCallHostChanged(serverUrl, msg); break; case WebsocketEvents.CALLS_USER_DISMISSED_NOTIFICATION: handleUserDismissedNotification(serverUrl, msg); break; + case WebsocketEvents.CALLS_CAPTION: + handleCallCaption(serverUrl, msg); + break; case WebsocketEvents.GROUP_RECEIVED: handleGroupReceivedEvent(serverUrl, msg); diff --git a/app/constants/calls.ts b/app/constants/calls.ts index 46003e170..027d02f94 100644 --- a/app/constants/calls.ts +++ b/app/constants/calls.ts @@ -24,12 +24,18 @@ const PluginId = 'com.mattermost.calls'; const REACTION_TIMEOUT = 10000; const REACTION_LIMIT = 20; const CALL_QUALITY_RESET_MS = toMilliseconds({minutes: 1}); +const CAPTION_TIMEOUT = 5000; export enum MessageBarType { Microphone, CallQuality, } +// The JobTypes from calls plugin's server/public/job.go +const JOB_TYPE_RECORDING = 'recording'; +const JOB_TYPE_TRANSCRIBING = 'transcribing'; +const JOB_TYPE_CAPTIONING = 'captioning'; + export default { RefreshConfigMillis, RequiredServer, @@ -39,4 +45,8 @@ export default { REACTION_LIMIT, MessageBarType, CALL_QUALITY_RESET_MS, + CAPTION_TIMEOUT, + JOB_TYPE_RECORDING, + JOB_TYPE_TRANSCRIBING, + JOB_TYPE_CAPTIONING, }; diff --git a/app/constants/websocket.ts b/app/constants/websocket.ts index 51c745284..906ae15ea 100644 --- a/app/constants/websocket.ts +++ b/app/constants/websocket.ts @@ -81,9 +81,14 @@ const WebsocketEvents = { CALLS_USER_RAISE_HAND: `custom_${Calls.PluginId}_user_raise_hand`, CALLS_USER_UNRAISE_HAND: `custom_${Calls.PluginId}_user_unraise_hand`, CALLS_USER_REACTED: `custom_${Calls.PluginId}_user_reacted`, + + // DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0) CALLS_RECORDING_STATE: `custom_${Calls.PluginId}_call_recording_state`, + CALLS_JOB_STATE: `custom_${Calls.PluginId}_call_job_state`, CALLS_HOST_CHANGED: `custom_${Calls.PluginId}_call_host_changed`, CALLS_USER_DISMISSED_NOTIFICATION: `custom_${Calls.PluginId}_user_dismissed_notification`, + CALLS_CAPTION: `custom_${Calls.PluginId}_caption`, + GROUP_RECEIVED: 'received_group', GROUP_MEMBER_ADD: 'group_member_add', GROUP_MEMBER_DELETE: 'group_member_delete', diff --git a/app/products/calls/client/rest.ts b/app/products/calls/client/rest.ts index 423ad7388..6e3fb85d5 100644 --- a/app/products/calls/client/rest.ts +++ b/app/products/calls/client/rest.ts @@ -2,7 +2,7 @@ // See LICENSE.txt for license information. import type {ApiResp, CallsVersion} from '@calls/types/calls'; -import type {CallChannelState, CallRecordingState, CallsConfig} from '@mattermost/calls/lib/types'; +import type {CallChannelState, CallJobState, CallsConfig} from '@mattermost/calls/lib/types'; import type {RTCIceServer} from 'react-native-webrtc'; export interface ClientCallsMix { @@ -14,8 +14,8 @@ export interface ClientCallsMix { enableChannelCalls: (channelId: string, enable: boolean) => Promise; endCall: (channelId: string) => Promise; genTURNCredentials: () => Promise; - startCallRecording: (callId: string) => Promise; - stopCallRecording: (callId: string) => Promise; + startCallRecording: (callId: string) => Promise; + stopCallRecording: (callId: string) => Promise; dismissCall: (channelId: string) => Promise; } diff --git a/app/products/calls/components/captions.tsx b/app/products/calls/components/captions.tsx new file mode 100644 index 000000000..47bffee41 --- /dev/null +++ b/app/products/calls/components/captions.tsx @@ -0,0 +1,119 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import React, {useEffect, useState} from 'react'; +import {useIntl} from 'react-intl'; +import {StyleSheet, Text, View} from 'react-native'; + +import CompassIcon from '@components/compass_icon'; +import FormattedText from '@components/formatted_text'; +import {changeOpacity} from '@utils/theme'; +import {displayUsername} from '@utils/user'; + +import type {CallSession, LiveCaptionMobile} from '@calls/types/calls'; + +const styles = StyleSheet.create({ + spacingContainer: { + position: 'relative', + width: '90%', + height: 48, + }, + captionContainer: { + display: 'flex', + height: 400, + bottom: -352, // 48-400, to place the bottoms at the same place + gap: 8, + alignItems: 'center', + flexDirection: 'column-reverse', + overflow: 'hidden', + }, + caption: { + paddingTop: 1, + paddingRight: 8, + paddingBottom: 3, + paddingLeft: 8, + borderRadius: 4, + backgroundColor: changeOpacity('#000', 0.64), + }, + captionNotice: { + display: 'flex', + flexDirection: 'row', + gap: 8, + }, + text: { + color: '#FFF', + fontFamily: 'Open Sans', + fontSize: 16, + fontStyle: 'normal', + fontWeight: '400', + lineHeight: 22, + textAlign: 'center', + }, +}); + +type Props = { + captionsDict: Dictionary; + sessionsDict: Dictionary; + teammateNameDisplay: string; +} + +const Captions = ({captionsDict, sessionsDict, teammateNameDisplay}: Props) => { + const intl = useIntl(); + const [showCCNotice, setShowCCNotice] = useState(true); + + useEffect(() => { + const timeoutID = setTimeout(() => { + setShowCCNotice(false); + }, 2000); + return () => clearTimeout(timeoutID); + }, []); + + const captionsArr = Object.values(captionsDict).reverse(); + + if (showCCNotice && captionsArr.length > 0) { + setShowCCNotice(false); + } + if (showCCNotice) { + return ( + + + + + + + + + ); + } + + return ( + + + {captionsArr.map((cap) => ( + + + {`(${displayUsername(sessionsDict[cap.sessionId]?.userModel, intl.locale, teammateNameDisplay)}) ${cap.text}`} + + + ))} + + + ); +}; + +export default Captions; diff --git a/app/products/calls/connection/websocket_event_handlers.ts b/app/products/calls/connection/websocket_event_handlers.ts index 69322dae8..6986ef6a2 100644 --- a/app/products/calls/connection/websocket_event_handlers.ts +++ b/app/products/calls/connection/websocket_event_handlers.ts @@ -6,10 +6,13 @@ import {DeviceEventEmitter} from 'react-native'; import {fetchUsersByIds} from '@actions/remote/user'; import { callEnded, - callStarted, getCallsConfig, + callStarted, + getCallsConfig, + receivedCaption, removeIncomingCall, setCallScreenOff, setCallScreenOn, + setCaptioningState, setChannelEnabled, setHost, setRaisedHand, @@ -22,14 +25,18 @@ import { } from '@calls/state'; import {isMultiSessionSupported} from '@calls/utils'; import {WebsocketEvents} from '@constants'; +import Calls from '@constants/calls'; import DatabaseManager from '@database/manager'; import {getCurrentUserId} from '@queries/servers/system'; +import type {CallRecordingStateData} from '@calls/types/calls'; import type { CallHostChangedData, - CallRecordingStateData, + CallJobState, + CallJobStateData, CallStartData, EmptyData, + LiveCaptionData, UserConnectedData, UserDisconnectedData, UserDismissedNotification, @@ -155,8 +162,24 @@ export const handleCallUserReacted = (serverUrl: string, msg: WebSocketMessage) => { - setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.recState); + const jobState: CallJobState = { + type: Calls.JOB_TYPE_RECORDING, + ...msg.data.recState, + }; + setRecordingState(serverUrl, msg.broadcast.channel_id, jobState); +}; + +export const handleCallJobState = (serverUrl: string, msg: WebSocketMessage) => { + switch (msg.data.jobState.type) { + case Calls.JOB_TYPE_RECORDING: + setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.jobState); + break; + case Calls.JOB_TYPE_CAPTIONING: + setCaptioningState(serverUrl, msg.broadcast.channel_id, msg.data.jobState); + break; + } }; export const handleCallHostChanged = (serverUrl: string, msg: WebSocketMessage) => { @@ -177,3 +200,7 @@ export const handleUserDismissedNotification = async (serverUrl: string, msg: We removeIncomingCall(serverUrl, msg.data.callID); }; + +export const handleCallCaption = (serverUrl: string, msg: WebSocketMessage) => { + receivedCaption(serverUrl, msg.data); +}; diff --git a/app/products/calls/screens/call_screen/call_screen.tsx b/app/products/calls/screens/call_screen/call_screen.tsx index 322415d17..f914ea786 100644 --- a/app/products/calls/screens/call_screen/call_screen.tsx +++ b/app/products/calls/screens/call_screen/call_screen.tsx @@ -1,5 +1,6 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. +/* eslint max-lines: off */ import React, {useCallback, useEffect, useMemo, useState} from 'react'; import {useIntl} from 'react-intl'; @@ -29,6 +30,7 @@ import CallAvatar from '@calls/components/call_avatar'; import CallDuration from '@calls/components/call_duration'; import CallNotification from '@calls/components/call_notification'; import CallsBadge, {CallsBadgeType} from '@calls/components/calls_badge'; +import Captions from '@calls/components/captions'; import EmojiList from '@calls/components/emoji_list'; import MessageBar from '@calls/components/message_bar'; import ReactionBar from '@calls/components/reaction_bar'; @@ -334,6 +336,7 @@ const CallScreen = ({ const [showControlsInLandscape, setShowControlsInLandscape] = useState(false); const [showReactions, setShowReactions] = useState(false); + const [showCC, setShowCC] = useState(false); const callsTheme = useMemo(() => makeCallsTheme(theme), [theme]); const style = getStyleSheet(callsTheme); const [centerUsers, setCenterUsers] = useState(false); @@ -343,7 +346,7 @@ const CallScreen = ({ const micPermissionsError = !micPermissionsGranted && !currentCall?.micPermissionsErrorDismissed; const screenShareOn = Boolean(currentCall?.screenOn); const isLandscape = width > height; - const smallerAvatar = isLandscape || screenShareOn; + const smallerAvatar = isLandscape || screenShareOn || showCC; const avatarSize = smallerAvatar ? avatarM : avatarL; const numSessions = Object.keys(sessionsDict).length; const showIncomingCalls = incomingCalls.incomingCalls.length > 0; @@ -358,6 +361,8 @@ const CallScreen = ({ id: 'mobile.calls_open_channel', defaultMessage: 'Open Channel', }); + const showCCTitle = intl.formatMessage({id: 'mobile.calls_show_cc', defaultMessage: 'Show live captions'}); + const hideCCTitle = intl.formatMessage({id: 'mobile.calls_hide_cc', defaultMessage: 'Hide live captions'}); useEffect(() => { mergeNavigationOptions('Call', { @@ -424,6 +429,13 @@ const CallScreen = ({ await stopCallRecording(currentCall.serverUrl, currentCall.channelId); }, [currentCall?.channelId, currentCall?.serverUrl]); + const toggleCC = useCallback(async () => { + Keyboard.dismiss(); + await dismissBottomSheet(); + + setShowCC((prev) => !prev); + }, [setShowCC]); + const switchToThread = useCallback(async () => { Keyboard.dismiss(); await dismissBottomSheet(); @@ -472,6 +484,7 @@ const CallScreen = ({ const waitingForRecording = Boolean(currentCall?.recState?.init_at && !currentCall.recState.start_at && !currentCall.recState.end_at && isHost); const showStartRecording = isHost && EnableRecordings && !(waitingForRecording || recording); const showStopRecording = isHost && EnableRecordings && (waitingForRecording || recording); + const ccAvailable = Boolean((currentCall?.capState?.start_at || 0) > (currentCall?.capState?.end_at || 0)); const showOtherActions = useCallback(async () => { const renderContent = () => { @@ -499,11 +512,22 @@ const CallScreen = ({ onPress={switchToThread} text={callThreadOptionTitle} /> + { + ccAvailable && + + } ); }; - const items = isHost && EnableRecordings ? 3 : 2; + let items = isHost && EnableRecordings ? 3 : 2; + if (ccAvailable) { + items++; + } bottomSheet({ closeButtonId: 'close-other-actions', renderContent, @@ -512,8 +536,9 @@ const CallScreen = ({ theme, }); }, [bottom, intl, theme, isHost, EnableRecordings, waitingForRecording, recording, startRecording, - recordOptionTitle, stopRecording, stopRecordingOptionTitle, style, switchToThread, callThreadOptionTitle, - openChannelOptionTitle]); + recordOptionTitle, stopRecording, stopRecordingOptionTitle, style, switchToThread, + callThreadOptionTitle, openChannelOptionTitle, ccAvailable, toggleCC, showCC, hideCCTitle, + showCCTitle]); const collapse = useCallback(() => { popTopScreen(componentId); @@ -693,6 +718,13 @@ const CallScreen = ({ {usersList} {screenShareView} {isLandscape && header} + {showCC && + + } {!isLandscape && currentCall.reactionStream.length > 0 && } @@ -789,7 +821,7 @@ const CallScreen = ({ style={style.buttonText} /> - {!isLandscape && isHost && + {!isLandscape && (isHost || ccAvailable) && } - {(isLandscape || !isHost) && + {(isLandscape || (!isHost && !ccAvailable)) && {stopRecordingOptionTitle} } + {isLandscape && ccAvailable && + + + + + } diff --git a/app/products/calls/state/actions.test.ts b/app/products/calls/state/actions.test.ts index db5d2d2f9..e1bd007eb 100644 --- a/app/products/calls/state/actions.test.ts +++ b/app/products/calls/state/actions.test.ts @@ -10,6 +10,7 @@ import { newCurrentCall, processIncomingCalls, processMeanOpinionScore, + receivedCaption, removeIncomingCall, setAudioDeviceInfo, setCallQualityAlertDismissed, @@ -60,9 +61,10 @@ import { type GlobalCallsState, } from '@calls/types/calls'; import {License} from '@constants'; +import Calls from '@constants/calls'; import DatabaseManager from '@database/manager'; -import type {CallRecordingState} from '@mattermost/calls/lib/types'; +import type {CallJobState, LiveCaptionData} from '@mattermost/calls/lib/types'; jest.mock('@calls/alerts'); @@ -1202,7 +1204,8 @@ describe('useCallsState', () => { myUserId: 'myUserId', ...call1, }; - const recState: CallRecordingState = { + const recState: CallJobState = { + type: Calls.JOB_TYPE_RECORDING, init_at: 123, start_at: 231, end_at: 345, @@ -1407,4 +1410,81 @@ describe('useCallsState', () => { assert.deepEqual(result.current[1], currentCallStateImIn); assert.deepEqual(result.current[2], expectedIncomingCalls); }); + + it('captions', () => { + const initialCallsState = { + ...DefaultCallsState, + serverUrl: 'server1', + myUserId: 'myUserId', + calls: {'channel-1': call1, 'channel-2': call2}, + }; + const initialCurrentCallState: CurrentCall = { + ...DefaultCurrentCall, + serverUrl: 'server1', + myUserId: 'myUserId', + ...call1, + }; + const caption1user1Data: LiveCaptionData = { + session_id: 'session1', + user_id: 'user-1', + channel_id: 'channel-1', + text: 'caption 1', + }; + const caption2user1Data: LiveCaptionData = { + session_id: 'session1', + user_id: 'user-1', + channel_id: 'channel-1', + text: 'caption 2', + }; + const caption3user1Data: LiveCaptionData = { + session_id: 'session1', + user_id: 'user-1', + channel_id: 'channel-1', + text: 'caption 3', + }; + const caption1user2Data: LiveCaptionData = { + session_id: 'session2', + user_id: 'user-2', + channel_id: 'channel-1', + text: 'caption 1 user 2', + }; + const caption2user2Data: LiveCaptionData = { + session_id: 'session2', + user_id: 'user-2', + channel_id: 'channel-1', + text: 'caption 2 user 2', + }; + + // setup + const {result} = renderHook(() => { + return [useCallsState('server1'), useCurrentCall()]; + }); + act(() => { + setCallsState('server1', initialCallsState); + setCurrentCall(initialCurrentCallState); + }); + assert.deepEqual(result.current[0], initialCallsState); + assert.deepEqual(result.current[1], initialCurrentCallState); + + // test sending the first 2 captions for user 1, 1 caption for user 2 + act(() => { + receivedCaption('server1', caption1user1Data); + receivedCaption('server1', caption2user1Data); + receivedCaption('server1', caption1user2Data); + }); + assert.deepEqual(result.current[0], initialCallsState); + let currentCall = result.current[1] as CurrentCall; + assert.equal(currentCall.captions.session1.text, 'caption 2'); + assert.equal(currentCall.captions.session2.text, 'caption 1 user 2'); + + // test sending the next captions for users 1 and 2 + act(() => { + receivedCaption('server1', caption3user1Data); + receivedCaption('server1', caption2user2Data); + }); + assert.deepEqual(result.current[0], initialCallsState); + currentCall = result.current[1] as CurrentCall; + assert.equal(currentCall.captions.session1.text, 'caption 3'); + assert.equal(currentCall.captions.session2.text, 'caption 2 user 2'); + }); }); diff --git a/app/products/calls/state/actions.ts b/app/products/calls/state/actions.ts index ea1d14498..5ce391ec4 100644 --- a/app/products/calls/state/actions.ts +++ b/app/products/calls/state/actions.ts @@ -30,6 +30,7 @@ import { DefaultCall, DefaultCurrentCall, type IncomingCallNotification, + type LiveCaptionMobile, type ReactionStreamEmoji, } from '@calls/types/calls'; import {Calls, General, Screens} from '@constants'; @@ -38,9 +39,10 @@ import {getChannelById} from '@queries/servers/channel'; import {getThreadById} from '@queries/servers/thread'; import {getUserById} from '@queries/servers/user'; import {isDMorGM} from '@utils/channel'; +import {generateId} from '@utils/general'; import {logDebug} from '@utils/log'; -import type {CallRecordingState, UserReactionData} from '@mattermost/calls/lib/types'; +import type {CallJobState, LiveCaptionData, UserReactionData} from '@mattermost/calls/lib/types'; export const setCalls = async (serverUrl: string, myUserId: string, calls: Dictionary, enabled: Dictionary) => { const channelsWithCalls = Object.keys(calls).reduce( @@ -678,7 +680,7 @@ const userReactionTimeout = (serverUrl: string, channelId: string, reaction: Use setCurrentCall(nextCurrentCall); }; -export const setRecordingState = (serverUrl: string, channelId: string, recState: CallRecordingState) => { +export const setRecordingState = (serverUrl: string, channelId: string, recState: CallJobState) => { const callsState = getCallsState(serverUrl); if (!callsState.calls[channelId]) { return; @@ -706,6 +708,29 @@ export const setRecordingState = (serverUrl: string, channelId: string, recState setCurrentCall(nextCurrentCall); }; +export const setCaptioningState = (serverUrl: string, channelId: string, capState: CallJobState) => { + const callsState = getCallsState(serverUrl); + if (!callsState.calls[channelId]) { + return; + } + + const nextCall = {...callsState.calls[channelId], capState}; + const nextCalls = {...callsState.calls, [channelId]: nextCall}; + setCallsState(serverUrl, {...callsState, calls: nextCalls}); + + // Was it the current call? If so, update that too. + const currentCall = getCurrentCall(); + if (!currentCall || currentCall.channelId !== channelId) { + return; + } + + const nextCurrentCall = { + ...currentCall, + capState, + }; + setCurrentCall(nextCurrentCall); +}; + export const setHost = (serverUrl: string, channelId: string, hostId: string) => { const callsState = getCallsState(serverUrl); if (!callsState.calls[channelId]) { @@ -784,3 +809,55 @@ export const setCallQualityAlertDismissed = () => { }; setCurrentCall(nextCurrentCall); }; + +export const receivedCaption = (serverUrl: string, captionData: LiveCaptionData) => { + const channelId = captionData.channel_id; + + // Ignore if we're not in that channel's call. + const currentCall = getCurrentCall(); + if (currentCall?.channelId !== channelId) { + return; + } + + // Add or replace that user's caption. + const captionId = generateId(); + const nextCaptions = {...currentCall.captions}; + const newCaption: LiveCaptionMobile = { + captionId, + sessionId: captionData.session_id, + userId: captionData.user_id, + text: captionData.text, + }; + nextCaptions[captionData.session_id] = newCaption; + + const nextCurrentCall: CurrentCall = { + ...currentCall, + captions: nextCaptions, + }; + setCurrentCall(nextCurrentCall); + + setTimeout(() => { + receivedCaptionTimeout(serverUrl, channelId, newCaption); + }, Calls.CAPTION_TIMEOUT); +}; + +const receivedCaptionTimeout = (serverUrl: string, channelId: string, caption: LiveCaptionMobile) => { + const currentCall = getCurrentCall(); + if (currentCall?.channelId !== channelId) { + return; + } + + // Remove the caption only if it hasn't been replaced by a newer one + if (currentCall.captions[caption.sessionId]?.captionId !== caption.captionId) { + return; + } + + const nextCaptions = {...currentCall.captions}; + delete nextCaptions[caption.sessionId]; + + const nextCurrentCall: CurrentCall = { + ...currentCall, + captions: nextCaptions, + }; + setCurrentCall(nextCurrentCall); +}; diff --git a/app/products/calls/types/calls.ts b/app/products/calls/types/calls.ts index 562cea70e..1886372b2 100644 --- a/app/products/calls/types/calls.ts +++ b/app/products/calls/types/calls.ts @@ -1,7 +1,12 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import type {CallRecordingState, CallsConfig, EmojiData, UserReactionData} from '@mattermost/calls/lib/types'; +import type { + CallJobState, + CallsConfig, + EmojiData, + UserReactionData, +} from '@mattermost/calls/lib/types'; import type UserModel from '@typings/database/models/servers/user'; export type GlobalCallsState = { @@ -56,7 +61,8 @@ export type Call = { screenOn: string; threadId: string; ownerId: string; - recState?: CallRecordingState; + recState?: CallJobState; + capState?: CallJobState; hostId: string; dismissed: Dictionary; } @@ -94,6 +100,7 @@ export type CurrentCall = Call & { reactionStream: ReactionStreamEmoji[]; callQualityAlert: boolean; callQualityAlertDismissed: number; + captions: Dictionary; } export const DefaultCurrentCall: CurrentCall = { @@ -110,6 +117,7 @@ export const DefaultCurrentCall: CurrentCall = { reactionStream: [], callQualityAlert: false, callQualityAlertDismissed: 0, + captions: {}, }; export type CallSession = { @@ -158,6 +166,7 @@ export const DefaultCallsConfig: CallsConfigState = { EnableSimulcast: false, EnableRinging: false, EnableTranscriptions: false, + EnableLiveCaptions: false, }; export type ApiResp = { @@ -205,3 +214,24 @@ export type SelectedSubtitleTrack = { type: 'system' | 'disabled' | 'title' | 'language' | 'index'; value?: string | number | undefined; }; + +export type LiveCaptionMobile = { + captionId: string; + sessionId: string; + userId: string; + text: string; +} + +// DEPRECATED in favour of CallJobState since v2.16 +export type CallRecordingState = { + init_at: number; + start_at: number; + end_at: number; + err?: string; + error_at?: number; +} + +export type CallRecordingStateData = { + recState: CallRecordingState; + callID: string; +} diff --git a/assets/base/i18n/en.json b/assets/base/i18n/en.json index 7db1a64a3..3834ae745 100644 --- a/assets/base/i18n/en.json +++ b/assets/base/i18n/en.json @@ -444,6 +444,8 @@ "mobile.calls_call_ended": "Call ended", "mobile.calls_call_screen": "Call", "mobile.calls_call_thread": "Call Thread", + "mobile.calls_captions": "Captions", + "mobile.calls_captions_turned_on": "Live captions turned on", "mobile.calls_current_call": "Current call", "mobile.calls_disable": "Disable calls", "mobile.calls_dismiss": "Dismiss", @@ -458,6 +460,7 @@ "mobile.calls_error_message": "Error: {error}", "mobile.calls_error_title": "Error", "mobile.calls_headset": "Headset", + "mobile.calls_hide_cc": "Hide live captions", "mobile.calls_host": "host", "mobile.calls_host_rec": "You are recording this meeting. Consider letting everyone know that this meeting is being recorded.", "mobile.calls_host_rec_error": "Please try to record again. You can also contact your system admin for troubleshooting help.", @@ -509,6 +512,7 @@ "mobile.calls_request_message": "Calls are currently running in test mode and only system admins can start them. Reach out directly to your system admin for assistance", "mobile.calls_request_title": "Calls is not currently enabled", "mobile.calls_see_logs": "See server logs", + "mobile.calls_show_cc": "Show live captions", "mobile.calls_speaker": "Speaker", "mobile.calls_start_call": "Start Call", "mobile.calls_start_call_exists": "A call is already ongoing in the channel.", @@ -779,7 +783,6 @@ "notification_settings.mention.reply": "Send reply notifications for", "notification_settings.mentions": "Mentions", "notification_settings.mentions_replies": "Mentions and Replies", - "notification_settings.mentions..keywordsDescription": "Other words that trigger a mention", "notification_settings.mentions.channelWide": "Channel-wide mentions", "notification_settings.mentions.keywords": "Enter other keywords", "notification_settings.mentions.keywords_mention": "Keywords that trigger mentions", diff --git a/ios/Mattermost/Info.plist b/ios/Mattermost/Info.plist index b6753efcc..0d8433580 100644 --- a/ios/Mattermost/Info.plist +++ b/ios/Mattermost/Info.plist @@ -97,6 +97,7 @@ audio fetch remote-notification + voip UILaunchStoryboardName LaunchScreen diff --git a/package-lock.json b/package-lock.json index ed0a835b6..c818f5f2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,8 +18,8 @@ "@formatjs/intl-pluralrules": "5.2.10", "@formatjs/intl-relativetimeformat": "11.2.10", "@gorhom/bottom-sheet": "4.5.1", - "@mattermost/calls": "github:mattermost/calls-common#v0.22.0", - "@mattermost/compass-icons": "0.1.40", + "@mattermost/calls": "github:mattermost/calls-common#954d125d14fd3b46d723bab01d92a0ef3ecca1b8", + "@mattermost/compass-icons": "0.1.43", "@mattermost/react-native-emm": "1.4.0", "@mattermost/react-native-network-client": "1.5.0", "@mattermost/react-native-paste-input": "0.7.0", @@ -3446,7 +3446,8 @@ "node_modules/@mattermost/calls": { "name": "@calls/common", "version": "0.14.0", - "resolved": "git+ssh://git@github.com/mattermost/calls-common.git#4a45138c02e3ce45d9e26f81c0f421a136ae0e59" + "resolved": "git+ssh://git@github.com/mattermost/calls-common.git#954d125d14fd3b46d723bab01d92a0ef3ecca1b8", + "integrity": "sha512-dPuvLiWc33WhhTTDqqcurstsh7A3VoGhiAHugQ5rWidvrG1OkzshzTj3jJ0dlCGIUR9EORe+m7M+XeyGgR6KvA==" }, "node_modules/@mattermost/commonmark": { "version": "0.30.1-2", @@ -3468,9 +3469,9 @@ } }, "node_modules/@mattermost/compass-icons": { - "version": "0.1.40", - "resolved": "https://registry.npmjs.org/@mattermost/compass-icons/-/compass-icons-0.1.40.tgz", - "integrity": "sha512-Vz4BeTuwk7bgcKNyOn8F3Q0WsZF8s44bJNawg29pKQc4TLVPurceUhlsqi+B2vVTyCSVyNlFWijg/PWemyCgdQ==" + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/@mattermost/compass-icons/-/compass-icons-0.1.43.tgz", + "integrity": "sha512-vQThJ4SAynnS2u94lQtZ9xANsStpVh8uTpsJascHJOWcavLuL2aDmMLgvg9EAx8Z1qRmTdP6hF5+IU5+9E9+Jg==" }, "node_modules/@mattermost/react-native-emm": { "version": "1.4.0", diff --git a/package.json b/package.json index 123025ea2..4c2a1c783 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,8 @@ "@formatjs/intl-pluralrules": "5.2.10", "@formatjs/intl-relativetimeformat": "11.2.10", "@gorhom/bottom-sheet": "4.5.1", - "@mattermost/calls": "github:mattermost/calls-common#v0.22.0", - "@mattermost/compass-icons": "0.1.40", + "@mattermost/calls": "github:mattermost/calls-common#954d125d14fd3b46d723bab01d92a0ef3ecca1b8", + "@mattermost/compass-icons": "0.1.43", "@mattermost/react-native-emm": "1.4.0", "@mattermost/react-native-network-client": "1.5.0", "@mattermost/react-native-paste-input": "0.7.0",