1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.android.material.floatingactionbutton.FloatingActionButton;
36import com.google.common.base.Optional;
37import com.google.common.base.Preconditions;
38import com.google.common.base.Strings;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.CallIntegration;
52import eu.siacs.conversations.services.CallIntegrationConnectionService;
53import eu.siacs.conversations.services.XmppConnectionService;
54import eu.siacs.conversations.ui.util.AvatarWorkerTask;
55import eu.siacs.conversations.ui.util.MainThreadExecutor;
56import eu.siacs.conversations.ui.util.Rationals;
57import eu.siacs.conversations.utils.PermissionUtils;
58import eu.siacs.conversations.utils.TimeFrameUtils;
59import eu.siacs.conversations.xmpp.Jid;
60import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
61import eu.siacs.conversations.xmpp.jingle.ContentAddition;
62import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
63import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
64import eu.siacs.conversations.xmpp.jingle.Media;
65import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
66import eu.siacs.conversations.xmpp.jingle.RtpCapability;
67import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
68
69import org.webrtc.RendererCommon;
70import org.webrtc.SurfaceViewRenderer;
71import org.webrtc.VideoTrack;
72
73import java.lang.ref.WeakReference;
74import java.util.Arrays;
75import java.util.Collections;
76import java.util.List;
77import java.util.Set;
78
79public class RtpSessionActivity extends XmppActivity
80 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
81 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
82
83 public static final String EXTRA_WITH = "with";
84 public static final String EXTRA_SESSION_ID = "session_id";
85 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
86 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
87 public static final String EXTRA_LAST_ACTION = "last_action";
88 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
89 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
90 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
91
92 private static final int CALL_DURATION_UPDATE_INTERVAL = 250;
93 private static final int BUTTON_VISIBILITY_TIMEOUT = 10_000;
94
95 private static final List<RtpEndUserState> END_CARD =
96 Arrays.asList(
97 RtpEndUserState.APPLICATION_ERROR,
98 RtpEndUserState.SECURITY_ERROR,
99 RtpEndUserState.DECLINED_OR_BUSY,
100 RtpEndUserState.CONTACT_OFFLINE,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
103 RtpEndUserState.RETRACTED);
104 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
105 Arrays.asList(
106 RtpEndUserState.APPLICATION_ERROR,
107 RtpEndUserState.CONNECTIVITY_ERROR,
108 RtpEndUserState.SECURITY_ERROR);
109 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
110 Arrays.asList(
111 RtpEndUserState.CONNECTING,
112 RtpEndUserState.CONNECTED,
113 RtpEndUserState.RECONNECTING,
114 RtpEndUserState.INCOMING_CONTENT_ADD);
115 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
116 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
117 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
118 Arrays.asList(
119 RtpEndUserState.ACCEPTING_CALL,
120 RtpEndUserState.CONNECTING,
121 RtpEndUserState.RECONNECTING);
122 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
123 new ImmutableList.Builder<RtpEndUserState>()
124 .add(RtpEndUserState.FINDING_DEVICE)
125 .add(RtpEndUserState.RINGING)
126 .add(RtpEndUserState.ACCEPTING_CALL)
127 .add(RtpEndUserState.CONNECTING)
128 .addAll(STATES_CONSIDERED_CONNECTED)
129 .build();
130 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
131 private static final int REQUEST_ACCEPT_CALL = 0x1111;
132 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
133 private static final int REQUEST_ADD_CONTENT = 0x1113;
134 private WeakReference<JingleRtpConnection> rtpConnectionReference;
135
136 private ActivityRtpSessionBinding binding;
137 private PowerManager.WakeLock mProximityWakeLock;
138
139 private final Handler mHandler = new Handler();
140 private final Runnable mTickExecutor =
141 new Runnable() {
142 @Override
143 public void run() {
144 updateCallDuration();
145 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
146 }
147 };
148 private boolean buttonsHiddenAfterTimeout = false;
149 private final Runnable mVisibilityToggleExecutor = this::updateButtonInVideoCallVisibility;
150
151 public static Set<Media> actionToMedia(final String action) {
152 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
153 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
154 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
155 return ImmutableSet.of(Media.AUDIO);
156 } else {
157 Log.w(
158 Config.LOGTAG,
159 "actionToMedia can not get media set from unknown action " + action);
160 return Collections.emptySet();
161 }
162 }
163
164 private static void addSink(
165 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
166 try {
167 videoTrack.addSink(surfaceViewRenderer);
168 } catch (final IllegalStateException e) {
169 Log.e(
170 Config.LOGTAG,
171 "possible race condition on trying to display video track. ignoring",
172 e);
173 }
174 }
175
176 @Override
177 public void onCreate(Bundle savedInstanceState) {
178 super.onCreate(savedInstanceState);
179 getWindow()
180 .addFlags(
181 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
182 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
183 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
184 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
185 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
186 this.binding.remoteVideo.setOnClickListener(this::onVideoScreenClick);
187 this.binding.localVideo.setOnClickListener(this::onVideoScreenClick);
188 setSupportActionBar(binding.toolbar);
189 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
190 }
191
192 private void onVideoScreenClick(final View view) {
193 resetVisibilityExecutorShowButtons();
194 }
195
196 @Override
197 public boolean onCreateOptionsMenu(final Menu menu) {
198 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
199 final MenuItem help = menu.findItem(R.id.action_help);
200 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
201 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
202 help.setVisible(Config.HELP != null && isHelpButtonVisible());
203 gotoChat.setVisible(isSwitchToConversationVisible());
204 switchToVideo.setVisible(isSwitchToVideoVisible());
205 return super.onCreateOptionsMenu(menu);
206 }
207
208 @Override
209 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
210 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
211 if (xmppConnectionService != null) {
212 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
213 return true;
214 }
215 }
216 }
217 return super.onKeyDown(keyCode, event);
218 }
219
220 private boolean isHelpButtonVisible() {
221 try {
222 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
223 } catch (IllegalStateException e) {
224 final Intent intent = getIntent();
225 final String state =
226 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
227 if (state != null) {
228 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
229 } else {
230 return false;
231 }
232 }
233 }
234
235 private boolean isSwitchToConversationVisible() {
236 final JingleRtpConnection connection =
237 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
238 return connection != null
239 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
240 }
241
242 private boolean isSwitchToVideoVisible() {
243 final JingleRtpConnection connection =
244 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
245 if (connection == null) {
246 return false;
247 }
248 return connection.isSwitchToVideoAvailable();
249 }
250
251 private void switchToConversation() {
252 final Contact contact = getWith();
253 final Conversation conversation =
254 xmppConnectionService.findOrCreateConversation(
255 contact.getAccount(), contact.getJid(), false, true);
256 switchToConversation(conversation);
257 }
258
259 public boolean onOptionsItemSelected(final MenuItem item) {
260 final var itemItem = item.getItemId();
261 if (itemItem == R.id.action_help) {
262 launchHelpInBrowser();
263 return true;
264 } else if (itemItem == R.id.action_goto_chat) {
265 switchToConversation();
266 return true;
267 } else if (itemItem == R.id.action_switch_to_video) {
268 requestPermissionAndSwitchToVideo();
269 return true;
270 } else {
271 return super.onOptionsItemSelected(item);
272 }
273 }
274
275 private void launchHelpInBrowser() {
276 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
277 try {
278 startActivity(intent);
279 } catch (final ActivityNotFoundException e) {
280 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
281 .show();
282 }
283 }
284
285 private void endCall(View view) {
286 endCall();
287 }
288
289 private void endCall() {
290 if (this.rtpConnectionReference == null) {
291 retractSessionProposal();
292 finish();
293 } else {
294 requireRtpConnection().endCall();
295 }
296 }
297
298 private void retractSessionProposal() {
299 final Intent intent = getIntent();
300 final String action = intent.getAction();
301 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
302 final Account account = extractAccount(intent);
303 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
304 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
305 if (!Intent.ACTION_VIEW.equals(action)
306 || state == null
307 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
308 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
309 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
310 }
311 xmppConnectionService
312 .getJingleConnectionManager()
313 .retractSessionProposal(account, with.asBareJid());
314 }
315
316 private void rejectCall(View view) {
317 requireRtpConnection().rejectCall();
318 finish();
319 }
320
321 private void acceptCall(View view) {
322 requestPermissionsAndAcceptCall();
323 }
324
325 private void acceptContentAdd() {
326 try {
327 final ContentAddition pendingContentAddition =
328 requireRtpConnection().getPendingContentAddition();
329 if (pendingContentAddition == null) {
330 Log.d(Config.LOGTAG, "content offer was gone after granting permission");
331 return;
332 }
333 requireRtpConnection().acceptContentAdd(pendingContentAddition.summary);
334 } catch (final IllegalStateException e) {
335 Toast.makeText(this, Strings.nullToEmpty(e.getMessage()), Toast.LENGTH_SHORT).show();
336 }
337 }
338
339 private void requestPermissionAndSwitchToVideo() {
340 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
341 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
342 switchToVideo();
343 }
344 }
345
346 private void switchToVideo() {
347 try {
348 requireRtpConnection().addMedia(Media.VIDEO);
349 } catch (final IllegalStateException e) {
350 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
351 }
352 }
353
354 private void acceptContentAdd(final ContentAddition contentAddition) {
355 if (contentAddition == null
356 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
357 Log.d(Config.LOGTAG, "ignore press on content-accept button");
358 return;
359 }
360 requestPermissionAndAcceptContentAdd(contentAddition);
361 }
362
363 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
364 final List<String> permissions = permissions(contentAddition.media());
365 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
366 try {
367 requireRtpConnection().acceptContentAdd(contentAddition.summary);
368 } catch (final IllegalStateException e) {
369 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
370 }
371 }
372 }
373
374 private void rejectContentAdd(final View view) {
375 requireRtpConnection().rejectContentAdd();
376 }
377
378 private void requestPermissionsAndAcceptCall() {
379 final List<String> permissions = permissions(getMedia());
380 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
381 putScreenInCallMode();
382 acceptCall();
383 }
384 }
385
386 private List<String> permissions(final Set<Media> media) {
387 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
388 if (media.contains(Media.VIDEO)) {
389 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
390 } else {
391 permissions.add(Manifest.permission.RECORD_AUDIO);
392 }
393 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
394 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
395 }
396 return permissions.build();
397 }
398
399 private void acceptCall() {
400 try {
401 requireRtpConnection().acceptCall();
402 } catch (final IllegalStateException e) {
403 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
404 }
405 }
406
407 private void putScreenInCallMode() {
408 putScreenInCallMode(requireRtpConnection().getMedia());
409 }
410
411 private void putScreenInCallMode(final Set<Media> media) {
412 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
413 if (Media.audioOnly(media)) {
414 final JingleRtpConnection rtpConnection =
415 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
416 final CallIntegration callIntegration =
417 rtpConnection == null ? null : rtpConnection.getCallIntegration();
418 if (callIntegration == null
419 || callIntegration.getSelectedAudioDevice()
420 == CallIntegration.AudioDevice.EARPIECE) {
421 acquireProximityWakeLock();
422 }
423 }
424 lockOrientation(media);
425 }
426
427 private void lockOrientation(final Set<Media> media) {
428 if (Media.audioOnly(media)) {
429 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
430 } else {
431 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
432 }
433 }
434
435 @SuppressLint("WakelockTimeout")
436 private void acquireProximityWakeLock() {
437 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
438 if (powerManager == null) {
439 Log.e(Config.LOGTAG, "power manager not available");
440 return;
441 }
442 if (isFinishing()) {
443 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
444 return;
445 }
446 if (this.mProximityWakeLock == null) {
447 this.mProximityWakeLock =
448 powerManager.newWakeLock(
449 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
450 }
451 if (!this.mProximityWakeLock.isHeld()) {
452 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
453 this.mProximityWakeLock.acquire();
454 }
455 }
456
457 private void releaseProximityWakeLock() {
458 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
459 Log.d(Config.LOGTAG, "releasing proximity wake lock");
460 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
461 this.mProximityWakeLock = null;
462 }
463 }
464
465 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
466 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
467 acquireProximityWakeLock();
468 } else {
469 releaseProximityWakeLock();
470 }
471 }
472
473 @Override
474 protected void refreshUiReal() {}
475
476 @Override
477 public void onNewIntent(final Intent intent) {
478 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
479 super.onNewIntent(intent);
480 if (intent == null) {
481 return;
482 }
483 setIntent(intent);
484 if (xmppConnectionService == null) {
485 Log.d(
486 Config.LOGTAG,
487 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
488 return;
489 }
490 initializeWithIntent(Event.ON_NEW_INTENT, intent);
491 }
492
493 @Override
494 protected void onBackendConnected() {
495 final var intent = getIntent();
496 if (intent == null) {
497 return;
498 }
499 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
500 }
501
502 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
503 final String action = intent.getAction();
504 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
505 final Account account = extractAccount(intent);
506 final var extraWith = intent.getStringExtra(EXTRA_WITH);
507 final Jid with = Strings.isNullOrEmpty(extraWith) ? null : Jid.ofEscaped(extraWith);
508 if (with == null || account == null) {
509 Log.e(Config.LOGTAG, "intent is missing extras (account or with)");
510 return;
511 }
512 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
513 if (sessionId != null) {
514 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
515 return;
516 }
517 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
518 Log.d(Config.LOGTAG, "intent action was accept");
519 requestPermissionsAndAcceptCall();
520 resetIntent(intent.getExtras());
521 }
522 } else if (Intent.ACTION_VIEW.equals(action)) {
523 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
524 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
525 xmppConnectionService
526 .getJingleConnectionManager()
527 .getTerminalSessionState(with, proposedSessionId);
528 if (terminatedRtpSession != null) {
529 // termination (due to message error or 'busy' was faster than opening the activity
530 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
531 return;
532 }
533 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
534 final RtpEndUserState state =
535 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
536 final var contact = account.getRoster().getContact(with);
537 if (state != null) {
538 Log.d(Config.LOGTAG, "restored last state from intent extra");
539 updateButtonConfiguration(state);
540 updateVerifiedShield(false);
541 updateStateDisplay(state);
542 updateIncomingCallScreen(state);
543 updateSupportWarning(state, contact);
544 invalidateOptionsMenu();
545 }
546 setWith(state, contact);
547 if (xmppConnectionService
548 .getJingleConnectionManager()
549 .fireJingleRtpConnectionStateUpdates()) {
550 return;
551 }
552 if (END_CARD.contains(state)) {
553 return;
554 }
555 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
556 final Set<Media> media = actionToMedia(lastAction);
557 if (xmppConnectionService
558 .getJingleConnectionManager()
559 .hasMatchingProposal(account, with)) {
560 putScreenInCallMode(media);
561 return;
562 }
563 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
564 finish();
565 }
566 }
567
568 private void setWidth(final RtpEndUserState state) {
569 setWith(state, getWith());
570 }
571
572 private void setWith(final RtpEndUserState state, final Contact contact) {
573 binding.with.setText(contact.getDisplayName());
574 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
575 .contains(state)) {
576 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
577 binding.withJid.setVisibility(View.VISIBLE);
578 } else {
579 binding.withJid.setVisibility(View.GONE);
580 }
581 }
582
583 @Override
584 public void onRequestPermissionsResult(
585 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
586 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
587 final PermissionUtils.PermissionResult permissionResult =
588 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
589 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
590 if (requestCode == REQUEST_ACCEPT_CALL) {
591 acceptCall();
592 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
593 acceptContentAdd();
594 } else if (requestCode == REQUEST_ADD_CONTENT) {
595 switchToVideo();
596 }
597 } else {
598 @StringRes int res;
599 final String firstDenied =
600 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
601 if (firstDenied == null) {
602 return;
603 }
604 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
605 res = R.string.no_microphone_permission;
606 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
607 res = R.string.no_camera_permission;
608 } else {
609 throw new IllegalStateException("Invalid permission result request");
610 }
611 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
612 .show();
613 }
614 }
615
616 @Override
617 public void onStart() {
618 super.onStart();
619 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
620 mHandler.postDelayed(mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
621 this.binding.remoteVideo.setOnAspectRatioChanged(this);
622 }
623
624 @Override
625 public void onResume() {
626 super.onResume();
627 resetVisibilityExecutorShowButtons();
628 }
629
630 @Override
631 public void onStop() {
632 mHandler.removeCallbacks(mTickExecutor);
633 mHandler.removeCallbacks(mVisibilityToggleExecutor);
634 binding.remoteVideo.release();
635 binding.remoteVideo.setOnAspectRatioChanged(null);
636 binding.localVideo.release();
637 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
638 final JingleRtpConnection jingleRtpConnection =
639 weakReference == null ? null : weakReference.get();
640 if (jingleRtpConnection != null) {
641 releaseVideoTracks(jingleRtpConnection);
642 }
643 releaseProximityWakeLock();
644 super.onStop();
645 }
646
647 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
648 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
649 if (remoteVideo.isPresent()) {
650 remoteVideo.get().removeSink(binding.remoteVideo);
651 }
652 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
653 if (localVideo.isPresent()) {
654 localVideo.get().removeSink(binding.localVideo);
655 }
656 }
657
658 @Override
659 public void onBackPressed() {
660 if (isConnected()) {
661 if (switchToPictureInPicture()) {
662 return;
663 }
664 } else {
665 endCall();
666 }
667 super.onBackPressed();
668 }
669
670 @Override
671 public void onUserLeaveHint() {
672 super.onUserLeaveHint();
673 if (switchToPictureInPicture()) {
674 return;
675 }
676 // TODO apparently this method is not getting called on Android 10 when using the task
677 // switcher
678 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
679 retractSessionProposal();
680 }
681 }
682
683 private boolean isConnected() {
684 final JingleRtpConnection connection =
685 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
686 final RtpEndUserState endUserState =
687 connection == null ? null : connection.getEndUserState();
688 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
689 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
690 }
691
692 private boolean switchToPictureInPicture() {
693 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
694 if (shouldBePictureInPicture()) {
695 startPictureInPicture();
696 return true;
697 }
698 }
699 return false;
700 }
701
702 @RequiresApi(api = Build.VERSION_CODES.O)
703 private void startPictureInPicture() {
704 try {
705 final Rational rational = this.binding.remoteVideo.getAspectRatio();
706 final Rational clippedRational = Rationals.clip(rational);
707 Log.d(
708 Config.LOGTAG,
709 "suggested rational " + rational + ". clipped to " + clippedRational);
710 enterPictureInPictureMode(
711 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
712 } catch (final IllegalStateException e) {
713 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
714 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
715 }
716 }
717
718 @Override
719 public void onAspectRatioChanged(final Rational rational) {
720 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
721 final Rational clippedRational = Rationals.clip(rational);
722 Log.d(
723 Config.LOGTAG,
724 "suggested rational after aspect ratio change "
725 + rational
726 + ". clipped to "
727 + clippedRational);
728 setPictureInPictureParams(
729 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
730 }
731 }
732
733 private boolean deviceSupportsPictureInPicture() {
734 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
735 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
736 } else {
737 return false;
738 }
739 }
740
741 private boolean shouldBePictureInPicture() {
742 try {
743 final JingleRtpConnection rtpConnection = requireRtpConnection();
744 return rtpConnection.getMedia().contains(Media.VIDEO)
745 && Arrays.asList(
746 RtpEndUserState.ACCEPTING_CALL,
747 RtpEndUserState.CONNECTING,
748 RtpEndUserState.CONNECTED)
749 .contains(rtpConnection.getEndUserState());
750 } catch (final IllegalStateException e) {
751 return false;
752 }
753 }
754
755 private boolean isInConnectedVideoCall() {
756 final JingleRtpConnection rtpConnection;
757 try {
758 rtpConnection = requireRtpConnection();
759 } catch (final IllegalStateException e) {
760 return false;
761 }
762 return rtpConnection.getMedia().contains(Media.VIDEO)
763 && rtpConnection.getEndUserState() == RtpEndUserState.CONNECTED;
764 }
765
766 private boolean initializeActivityWithRunningRtpSession(
767 final Account account, Jid with, String sessionId) {
768 final WeakReference<JingleRtpConnection> reference =
769 xmppConnectionService
770 .getJingleConnectionManager()
771 .findJingleRtpConnection(account, with, sessionId);
772 if (reference == null || reference.get() == null) {
773 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
774 xmppConnectionService
775 .getJingleConnectionManager()
776 .getTerminalSessionState(with, sessionId);
777 if (terminatedRtpSession == null) {
778 throw new IllegalStateException(
779 "failed to initialize activity with running rtp session. session not found");
780 }
781 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
782 return true;
783 }
784 this.rtpConnectionReference = reference;
785 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
786 final boolean verified = requireRtpConnection().isVerified();
787 if (currentState == RtpEndUserState.ENDED) {
788 finish();
789 return true;
790 }
791 final Set<Media> media = getMedia();
792 final ContentAddition contentAddition = getPendingContentAddition();
793 if (currentState == RtpEndUserState.INCOMING_CALL) {
794 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
795 }
796 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
797 requireRtpConnection().getState())) {
798 putScreenInCallMode();
799 }
800 setWidth(currentState);
801 updateVideoViews(currentState);
802 updateStateDisplay(currentState, media, contentAddition);
803 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
804 updateButtonConfiguration(currentState, media, contentAddition);
805 updateIncomingCallScreen(currentState);
806 invalidateOptionsMenu();
807 return false;
808 }
809
810 private void initializeWithTerminatedSessionState(
811 final Account account,
812 final Jid with,
813 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
814 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
815 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
816 finish();
817 return;
818 }
819 final RtpEndUserState state = terminatedRtpSession.state;
820 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
821 updateButtonConfiguration(state);
822 updateStateDisplay(state);
823 updateIncomingCallScreen(state);
824 updateCallDuration();
825 updateVerifiedShield(false);
826 invalidateOptionsMenu();
827 final var contact = account.getRoster().getContact(with);
828 setWith(state, contact);
829 updateSupportWarning(state, contact);
830 }
831
832 private void reInitializeActivityWithRunningRtpSession(
833 final Account account, Jid with, String sessionId) {
834 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
835 resetIntent(account, with, sessionId);
836 }
837
838 private void resetIntent(final Account account, final Jid with, final String sessionId) {
839 final Intent intent = new Intent(Intent.ACTION_VIEW);
840 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
841 intent.putExtra(EXTRA_WITH, with.toEscapedString());
842 intent.putExtra(EXTRA_SESSION_ID, sessionId);
843 setIntent(intent);
844 }
845
846 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
847 surfaceViewRenderer.setVisibility(View.VISIBLE);
848 try {
849 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
850 } catch (final IllegalStateException ignored) {
851 // SurfaceViewRenderer was already initialized
852 } catch (final RuntimeException e) {
853 if (Throwables.getRootCause(e) instanceof GLException glException) {
854 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
855 }
856 }
857 surfaceViewRenderer.setEnableHardwareScaler(true);
858 }
859
860 private void updateStateDisplay(final RtpEndUserState state) {
861 updateStateDisplay(state, Collections.emptySet(), null);
862 }
863
864 private void updateStateDisplay(
865 final RtpEndUserState state,
866 final Set<Media> media,
867 final ContentAddition contentAddition) {
868 switch (state) {
869 case INCOMING_CALL -> {
870 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
871 if (media.contains(Media.VIDEO)) {
872 setTitle(R.string.rtp_state_incoming_video_call);
873 } else {
874 setTitle(R.string.rtp_state_incoming_call);
875 }
876 }
877 case INCOMING_CONTENT_ADD -> {
878 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
879 setTitle(R.string.rtp_state_content_add_video);
880 } else {
881 setTitle(R.string.rtp_state_content_add);
882 }
883 }
884 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
885 case CONNECTED -> setTitle(R.string.rtp_state_connected);
886 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
887 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
888 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
889 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
890 case RINGING -> setTitle(R.string.rtp_state_ringing);
891 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
892 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
893 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
894 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
895 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
896 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
897 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
898 case ENDED -> throw new IllegalStateException(
899 "Activity should have called finishAndReleaseWakeLock();");
900 default -> throw new IllegalStateException(
901 String.format("State %s has not been handled in UI", state));
902 }
903 }
904
905 private void updateVerifiedShield(final boolean verified) {
906 if (isPictureInPicture()) {
907 this.binding.verified.setVisibility(View.GONE);
908 return;
909 }
910 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
911 }
912
913 private void updateIncomingCallScreen(final RtpEndUserState state) {
914 updateIncomingCallScreen(state, null);
915 }
916
917 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
918 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
919 final boolean show = getResources().getBoolean(R.bool.is_portrait_mode);
920 if (show) {
921 binding.contactPhoto.setVisibility(View.VISIBLE);
922 if (contact == null) {
923 AvatarWorkerTask.loadAvatar(
924 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
925 } else {
926 AvatarWorkerTask.loadAvatar(
927 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
928 }
929 } else {
930 binding.contactPhoto.setVisibility(View.GONE);
931 }
932 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
933 binding.usingAccount.setVisibility(View.VISIBLE);
934 binding.usingAccount.setText(
935 getString(
936 R.string.using_account,
937 account.getJid().asBareJid().toEscapedString()));
938 } else {
939 binding.usingAccount.setVisibility(View.GONE);
940 binding.contactPhoto.setVisibility(View.GONE);
941 }
942 }
943
944 private void updateSupportWarning(final RtpEndUserState state, final Contact contact) {
945 if (state == RtpEndUserState.CONNECTIVITY_ERROR
946 && getResources().getBoolean(R.bool.is_portrait_mode)) {
947 binding.supportWarning.setVisibility(
948 RtpCapability.check(contact) == RtpCapability.Capability.NONE
949 ? View.VISIBLE
950 : View.GONE);
951 } else {
952 binding.supportWarning.setVisibility(View.GONE);
953 }
954 }
955
956 private Set<Media> getMedia() {
957 return requireRtpConnection().getMedia();
958 }
959
960 public ContentAddition getPendingContentAddition() {
961 return requireRtpConnection().getPendingContentAddition();
962 }
963
964 private void updateButtonConfiguration(final RtpEndUserState state) {
965 updateButtonConfiguration(state, Collections.emptySet(), null);
966 }
967
968 @SuppressLint("RestrictedApi")
969 private void updateButtonConfiguration(
970 final RtpEndUserState state,
971 final Set<Media> media,
972 final ContentAddition contentAddition) {
973 if (state == RtpEndUserState.ENDING_CALL
974 || isPictureInPicture()
975 || this.buttonsHiddenAfterTimeout) {
976 this.binding.rejectCall.setVisibility(View.INVISIBLE);
977 this.binding.endCall.setVisibility(View.INVISIBLE);
978 this.binding.acceptCall.setVisibility(View.INVISIBLE);
979 } else if (state == RtpEndUserState.INCOMING_CALL) {
980 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
981 this.binding.rejectCall.setOnClickListener(this::rejectCall);
982 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
983 this.binding.rejectCall.setVisibility(View.VISIBLE);
984 this.binding.endCall.setVisibility(View.INVISIBLE);
985 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
986 this.binding.acceptCall.setOnClickListener(this::acceptCall);
987 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
988 this.binding.acceptCall.setVisibility(View.VISIBLE);
989 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
990 this.binding.rejectCall.setContentDescription(
991 getString(R.string.reject_switch_to_video));
992 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
993 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
994 this.binding.rejectCall.setVisibility(View.VISIBLE);
995 this.binding.endCall.setVisibility(View.INVISIBLE);
996 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
997 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
998 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
999 this.binding.acceptCall.setVisibility(View.VISIBLE);
1000 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
1001 .contains(state)) {
1002 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1003 this.binding.rejectCall.setOnClickListener(this::exit);
1004 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1005 this.binding.rejectCall.setVisibility(View.VISIBLE);
1006 this.binding.endCall.setVisibility(View.INVISIBLE);
1007 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1008 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1009 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
1010 this.binding.acceptCall.setVisibility(View.VISIBLE);
1011 } else if (asList(
1012 RtpEndUserState.CONNECTIVITY_ERROR,
1013 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1014 RtpEndUserState.APPLICATION_ERROR,
1015 RtpEndUserState.RETRACTED,
1016 RtpEndUserState.SECURITY_ERROR)
1017 .contains(state)) {
1018 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1019 this.binding.rejectCall.setOnClickListener(this::exit);
1020 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1021 this.binding.rejectCall.setVisibility(View.VISIBLE);
1022 this.binding.endCall.setVisibility(View.INVISIBLE);
1023 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1024 this.binding.acceptCall.setOnClickListener(this::retry);
1025 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1026 this.binding.acceptCall.setVisibility(View.VISIBLE);
1027 } else {
1028 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1029 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1030 this.binding.endCall.setOnClickListener(this::endCall);
1031 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1032 setVisibleAndShow(this.binding.endCall);
1033 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1034 }
1035 updateInCallButtonConfiguration(state, media);
1036 }
1037
1038 private static void setVisibleAndShow(final FloatingActionButton button) {
1039 button.show();
1040 button.setVisibility(View.VISIBLE);
1041 }
1042
1043 private boolean isPictureInPicture() {
1044 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1045 return isInPictureInPictureMode();
1046 } else {
1047 return false;
1048 }
1049 }
1050
1051 private void updateInCallButtonConfiguration() {
1052 updateInCallButtonConfiguration(
1053 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1054 }
1055
1056 @SuppressLint("RestrictedApi")
1057 private void updateInCallButtonConfiguration(
1058 final RtpEndUserState state, final Set<Media> media) {
1059 final var showButtons = !isPictureInPicture() && !buttonsHiddenAfterTimeout;
1060 if (STATES_CONSIDERED_CONNECTED.contains(state) && showButtons) {
1061 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1062 if (media.contains(Media.VIDEO)) {
1063 final JingleRtpConnection rtpConnection = requireRtpConnection();
1064 updateInCallButtonConfigurationVideo(
1065 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1066 } else {
1067 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1068 updateInCallButtonConfigurationSpeaker(
1069 callIntegration.getSelectedAudioDevice(),
1070 callIntegration.getAudioDevices().size());
1071 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1072 }
1073 if (media.contains(Media.AUDIO)) {
1074 updateInCallButtonConfigurationMicrophone(
1075 requireRtpConnection().isMicrophoneEnabled());
1076 } else {
1077 this.binding.inCallActionLeft.setVisibility(View.GONE);
1078 }
1079 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1080 && showButtons
1081 && Media.audioOnly(media)) {
1082 final CallIntegration callIntegration;
1083 try {
1084 callIntegration = requireCallIntegration();
1085 } catch (final IllegalStateException e) {
1086 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1087 return;
1088 }
1089 updateInCallButtonConfigurationSpeaker(
1090 callIntegration.getSelectedAudioDevice(),
1091 callIntegration.getAudioDevices().size());
1092 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1093 } else {
1094 this.binding.inCallActionLeft.setVisibility(View.GONE);
1095 this.binding.inCallActionRight.setVisibility(View.GONE);
1096 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1097 }
1098 }
1099
1100 @SuppressLint("RestrictedApi")
1101 private void updateInCallButtonConfigurationSpeaker(
1102 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1103 switch (selectedAudioDevice) {
1104 case EARPIECE -> {
1105 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_24dp);
1106 if (numberOfChoices >= 2) {
1107 this.binding.inCallActionRight.setContentDescription(
1108 getString(R.string.call_is_using_earpiece_tap_to_switch_to_speaker));
1109 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1110 } else {
1111 this.binding.inCallActionRight.setContentDescription(
1112 getString(R.string.call_is_using_earpiece));
1113 this.binding.inCallActionRight.setOnClickListener(null);
1114 this.binding.inCallActionRight.setClickable(false);
1115 }
1116 }
1117 case WIRED_HEADSET -> {
1118 this.binding.inCallActionRight.setContentDescription(
1119 getString(R.string.call_is_using_wired_headset));
1120 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1121 this.binding.inCallActionRight.setOnClickListener(null);
1122 this.binding.inCallActionRight.setClickable(false);
1123 }
1124 case SPEAKER_PHONE -> {
1125 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1126 if (numberOfChoices >= 2) {
1127 this.binding.inCallActionRight.setContentDescription(
1128 getString(R.string.call_is_using_speaker_tap_to_switch_to_earpiece));
1129 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1130 } else {
1131 this.binding.inCallActionRight.setContentDescription(
1132 getString(R.string.call_is_using_speaker));
1133 this.binding.inCallActionRight.setOnClickListener(null);
1134 this.binding.inCallActionRight.setClickable(false);
1135 }
1136 }
1137 case BLUETOOTH -> {
1138 this.binding.inCallActionRight.setContentDescription(
1139 getString(R.string.call_is_using_bluetooth));
1140 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_24dp);
1141 this.binding.inCallActionRight.setOnClickListener(null);
1142 this.binding.inCallActionRight.setClickable(false);
1143 }
1144 }
1145 setVisibleAndShow(this.binding.inCallActionRight);
1146 }
1147
1148 @SuppressLint("RestrictedApi")
1149 private void updateInCallButtonConfigurationVideo(
1150 final boolean videoEnabled, final boolean isCameraSwitchable) {
1151 setVisibleAndShow(this.binding.inCallActionRight);
1152 if (isCameraSwitchable) {
1153 this.binding.inCallActionFarRight.setImageResource(
1154 R.drawable.ic_flip_camera_android_24dp);
1155 setVisibleAndShow(this.binding.inCallActionFarRight);
1156 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1157 this.binding.inCallActionFarRight.setContentDescription(
1158 getString(R.string.flip_camera));
1159 } else {
1160 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1161 }
1162 if (videoEnabled) {
1163 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1164 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1165 this.binding.inCallActionRight.setContentDescription(
1166 getString(R.string.video_is_enabled_tap_to_disable));
1167 } else {
1168 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1169 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1170 this.binding.inCallActionRight.setContentDescription(
1171 getString(R.string.video_is_disabled_tap_to_enable));
1172 }
1173 }
1174
1175 private void switchCamera(final View view) {
1176 resetVisibilityToggleExecutor();
1177 Futures.addCallback(
1178 requireRtpConnection().switchCamera(),
1179 new FutureCallback<>() {
1180 @Override
1181 public void onSuccess(@Nullable Boolean isFrontCamera) {
1182 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1183 }
1184
1185 @Override
1186 public void onFailure(@NonNull final Throwable throwable) {
1187 Log.d(
1188 Config.LOGTAG,
1189 "could not switch camera",
1190 Throwables.getRootCause(throwable));
1191 Toast.makeText(
1192 RtpSessionActivity.this,
1193 R.string.could_not_switch_camera,
1194 Toast.LENGTH_LONG)
1195 .show();
1196 }
1197 },
1198 MainThreadExecutor.getInstance());
1199 }
1200
1201 private void enableVideo(final View view) {
1202 resetVisibilityToggleExecutor();
1203 try {
1204 requireRtpConnection().setVideoEnabled(true);
1205 } catch (final IllegalStateException e) {
1206 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1207 return;
1208 }
1209 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1210 }
1211
1212 private void disableVideo(final View view) {
1213 resetVisibilityToggleExecutor();
1214 final JingleRtpConnection rtpConnection = requireRtpConnection();
1215 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1216 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1217 rtpConnection.retractContentAdd();
1218 return;
1219 }
1220 try {
1221 requireRtpConnection().setVideoEnabled(false);
1222 } catch (final IllegalStateException e) {
1223 Toast.makeText(this, R.string.could_not_disable_video, Toast.LENGTH_SHORT).show();
1224 return;
1225 }
1226 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1227 }
1228
1229 @SuppressLint("RestrictedApi")
1230 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1231 if (microphoneEnabled) {
1232 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1233 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1234 } else {
1235 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1236 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1237 }
1238 setVisibleAndShow(this.binding.inCallActionLeft);
1239 }
1240
1241 private void updateCallDuration() {
1242 final JingleRtpConnection connection =
1243 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1244 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1245 this.binding.duration.setVisibility(View.GONE);
1246 return;
1247 }
1248 if (connection.zeroDuration()) {
1249 this.binding.duration.setVisibility(View.GONE);
1250 } else {
1251 this.binding.duration.setText(
1252 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1253 this.binding.duration.setVisibility(View.VISIBLE);
1254 }
1255 }
1256
1257 private void resetVisibilityToggleExecutor() {
1258 mHandler.removeCallbacks(this.mVisibilityToggleExecutor);
1259 mHandler.postDelayed(this.mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
1260 }
1261
1262 private void updateButtonInVideoCallVisibility() {
1263 if (isInConnectedVideoCall()) {
1264 if (isPictureInPicture()) {
1265 return;
1266 }
1267 Log.d(Config.LOGTAG, "hiding in-call buttons after timeout was reached");
1268 hideInCallButtons();
1269 }
1270 }
1271
1272 private void hideInCallButtons() {
1273 binding.inCallActionLeft.hide();
1274 binding.endCall.hide();
1275 binding.inCallActionRight.hide();
1276 binding.inCallActionFarRight.hide();
1277 }
1278
1279 private void showInCallButtons() {
1280 this.buttonsHiddenAfterTimeout = false;
1281 final JingleRtpConnection rtpConnection;
1282 try {
1283 rtpConnection = requireRtpConnection();
1284 } catch (final IllegalStateException e) {
1285 return;
1286 }
1287 updateButtonConfiguration(
1288 rtpConnection.getEndUserState(),
1289 rtpConnection.getMedia(),
1290 rtpConnection.getPendingContentAddition());
1291 }
1292
1293 private void resetVisibilityExecutorShowButtons() {
1294 resetVisibilityToggleExecutor();
1295 showInCallButtons();
1296 }
1297
1298 private void updateVideoViews(final RtpEndUserState state) {
1299 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1300 binding.localVideo.setVisibility(View.GONE);
1301 binding.localVideo.release();
1302 binding.remoteVideoWrapper.setVisibility(View.GONE);
1303 binding.remoteVideo.release();
1304 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1305 if (isPictureInPicture()) {
1306 binding.appBarLayout.setVisibility(View.GONE);
1307 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1308 if (Arrays.asList(
1309 RtpEndUserState.APPLICATION_ERROR,
1310 RtpEndUserState.CONNECTIVITY_ERROR,
1311 RtpEndUserState.SECURITY_ERROR)
1312 .contains(state)) {
1313 binding.pipWarning.setVisibility(View.VISIBLE);
1314 binding.pipWaiting.setVisibility(View.GONE);
1315 } else {
1316 binding.pipWarning.setVisibility(View.GONE);
1317 binding.pipWaiting.setVisibility(View.GONE);
1318 }
1319 } else {
1320 binding.appBarLayout.setVisibility(View.VISIBLE);
1321 binding.pipPlaceholder.setVisibility(View.GONE);
1322 }
1323 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1324 return;
1325 }
1326 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1327 binding.localVideo.setVisibility(View.GONE);
1328 binding.remoteVideoWrapper.setVisibility(View.GONE);
1329 binding.appBarLayout.setVisibility(View.GONE);
1330 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1331 binding.pipWarning.setVisibility(View.GONE);
1332 binding.pipWaiting.setVisibility(View.VISIBLE);
1333 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1334 return;
1335 }
1336 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1337 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1338 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1339 // paint local view over remote view
1340 binding.localVideo.setZOrderMediaOverlay(true);
1341 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1342 addSink(localVideoTrack.get(), binding.localVideo);
1343 } else {
1344 binding.localVideo.setVisibility(View.GONE);
1345 }
1346 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1347 if (remoteVideoTrack.isPresent()) {
1348 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1349 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1350 binding.remoteVideo.setScalingType(
1351 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1352 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1353 if (state == RtpEndUserState.CONNECTED) {
1354 binding.appBarLayout.setVisibility(View.GONE);
1355 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1356 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1357 } else {
1358 binding.appBarLayout.setVisibility(View.VISIBLE);
1359 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1360 binding.remoteVideoWrapper.setVisibility(View.GONE);
1361 }
1362 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1363 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1364 } else {
1365 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1366 }
1367 } else {
1368 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1369 binding.remoteVideoWrapper.setVisibility(View.GONE);
1370 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1371 }
1372 }
1373
1374 private Optional<VideoTrack> getLocalVideoTrack() {
1375 final JingleRtpConnection connection =
1376 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1377 if (connection == null) {
1378 return Optional.absent();
1379 }
1380 return connection.getLocalVideoTrack();
1381 }
1382
1383 private Optional<VideoTrack> getRemoteVideoTrack() {
1384 final JingleRtpConnection connection =
1385 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1386 if (connection == null) {
1387 return Optional.absent();
1388 }
1389 return connection.getRemoteVideoTrack();
1390 }
1391
1392 private void disableMicrophone(final View view) {
1393 setMicrophoneEnabled(false);
1394 }
1395
1396 private void enableMicrophone(final View view) {
1397 setMicrophoneEnabled(true);
1398 }
1399
1400 private void setMicrophoneEnabled(final boolean enabled) {
1401 resetVisibilityExecutorShowButtons();
1402 try {
1403 final JingleRtpConnection rtpConnection = requireRtpConnection();
1404 if (rtpConnection.setMicrophoneEnabled(enabled)) {
1405 updateInCallButtonConfiguration();
1406 }
1407 } catch (final IllegalStateException e) {
1408 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1409 }
1410 }
1411
1412 private void switchToEarpiece(final View view) {
1413 try {
1414 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1415 acquireProximityWakeLock();
1416 } catch (final IllegalStateException e) {
1417 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1418 }
1419 }
1420
1421 private void switchToSpeaker(final View view) {
1422 try {
1423 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1424 releaseProximityWakeLock();
1425 } catch (final IllegalStateException e) {
1426 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1427 }
1428 }
1429
1430 private void retry(final View view) {
1431 final Intent intent = getIntent();
1432 final Account account = extractAccount(intent);
1433 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1434 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1435 final String action = intent.getAction();
1436 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1437 this.rtpConnectionReference = null;
1438 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1439 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1440 }
1441
1442 private void exit(final View view) {
1443 finish();
1444 }
1445
1446 private void recordVoiceMail(final View view) {
1447 final Intent intent = getIntent();
1448 final Account account = extractAccount(intent);
1449 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1450 final Conversation conversation =
1451 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1452 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1453 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1454 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1455 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1456 launchIntent.putExtra(
1457 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1458 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1459 startActivity(launchIntent);
1460 finish();
1461 }
1462
1463 private Contact getWith() {
1464 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1465 final Account account = id.account;
1466 return account.getRoster().getContact(id.with);
1467 }
1468
1469 private JingleRtpConnection requireRtpConnection() {
1470 final JingleRtpConnection connection =
1471 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1472 if (connection == null) {
1473 throw new IllegalStateException("No RTP connection found");
1474 }
1475 return connection;
1476 }
1477
1478 private CallIntegration requireCallIntegration() {
1479 return requireOngoingRtpSession().getCallIntegration();
1480 }
1481
1482 private OngoingRtpSession requireOngoingRtpSession() {
1483 final JingleRtpConnection connection =
1484 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1485 if (connection != null) {
1486 return connection;
1487 }
1488 final Intent currentIntent = getIntent();
1489 final String withExtra =
1490 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1491 final var account = extractAccount(currentIntent);
1492 if (withExtra == null) {
1493 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1494 }
1495 final var matching =
1496 xmppConnectionService
1497 .getJingleConnectionManager()
1498 .matchingProposal(account, Jid.of(withExtra));
1499 if (matching.isPresent()) {
1500 return matching.get();
1501 }
1502 throw new IllegalStateException("No matching session proposal");
1503 }
1504
1505 @Override
1506 public void onJingleRtpConnectionUpdate(
1507 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1508 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1509 if (END_CARD.contains(state)) {
1510 Log.d(Config.LOGTAG, "end card reached");
1511 releaseProximityWakeLock();
1512 runOnUiThread(
1513 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1514 }
1515 if (with.isBareJid()) {
1516 // TODO check for ENDED
1517 updateRtpSessionProposalState(account, with, state);
1518 return;
1519 }
1520 if (emptyReference(this.rtpConnectionReference)) {
1521 if (END_CARD.contains(state)) {
1522 Log.d(Config.LOGTAG, "not reinitializing session");
1523 return;
1524 }
1525 // this happens when going from proposed session to actual session
1526 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1527 return;
1528 }
1529 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1530 final boolean verified = requireRtpConnection().isVerified();
1531 final Set<Media> media = getMedia();
1532 lockOrientation(media);
1533 final ContentAddition contentAddition = getPendingContentAddition();
1534 final Contact contact = getWith();
1535 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1536 if (state == RtpEndUserState.ENDED) {
1537 finish();
1538 return;
1539 }
1540 resetVisibilityToggleExecutor();
1541 runOnUiThread(
1542 () -> {
1543 updateStateDisplay(state, media, contentAddition);
1544 updateVerifiedShield(
1545 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1546 updateButtonConfiguration(state, media, contentAddition);
1547 updateVideoViews(state);
1548 updateIncomingCallScreen(state, contact);
1549 updateSupportWarning(state, contact);
1550 invalidateOptionsMenu();
1551 });
1552 if (END_CARD.contains(state)) {
1553 final JingleRtpConnection rtpConnection = requireRtpConnection();
1554 resetIntent(account, with, state, rtpConnection.getMedia());
1555 releaseVideoTracks(rtpConnection);
1556 this.rtpConnectionReference = null;
1557 }
1558 } else {
1559 Log.d(Config.LOGTAG, "received update for other rtp session");
1560 }
1561 }
1562
1563 @Override
1564 public void onAudioDeviceChanged(
1565 final CallIntegration.AudioDevice selectedAudioDevice,
1566 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1567 Log.d(
1568 Config.LOGTAG,
1569 "onAudioDeviceChanged in activity: selected:"
1570 + selectedAudioDevice
1571 + ", available:"
1572 + availableAudioDevices);
1573 try {
1574 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1575 final RtpEndUserState endUserState;
1576 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1577 endUserState = jingleRtpConnection.getEndUserState();
1578 } else {
1579 // for session proposals all end user states are functionally the same
1580 endUserState = RtpEndUserState.RINGING;
1581 }
1582 final Set<Media> media = ongoingRtpSession.getMedia();
1583 if (END_CARD.contains(endUserState)) {
1584 Log.d(
1585 Config.LOGTAG,
1586 "onAudioDeviceChanged() nothing to do because end card has been reached");
1587 } else {
1588 if (Media.audioOnly(media)
1589 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1590 final CallIntegration callIntegration = requireCallIntegration();
1591 updateInCallButtonConfigurationSpeaker(
1592 callIntegration.getSelectedAudioDevice(),
1593 callIntegration.getAudioDevices().size());
1594 }
1595 Log.d(
1596 Config.LOGTAG,
1597 "put proximity wake lock into proper state after device update");
1598 putProximityWakeLockInProperState(selectedAudioDevice);
1599 }
1600 } catch (final IllegalStateException e) {
1601 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1602 }
1603 }
1604
1605 private void updateRtpSessionProposalState(
1606 final Account account, final Jid with, final RtpEndUserState state) {
1607 final Intent currentIntent = getIntent();
1608 final String withExtra =
1609 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1610 if (withExtra == null) {
1611 return;
1612 }
1613 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1614 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1615 runOnUiThread(
1616 () -> {
1617 updateVerifiedShield(false);
1618 updateStateDisplay(state);
1619 updateButtonConfiguration(state, media, null);
1620 updateIncomingCallScreen(state);
1621 updateSupportWarning(state, account.getRoster().getContact(with));
1622 invalidateOptionsMenu();
1623 });
1624 resetIntent(account, with, state, media);
1625 }
1626 }
1627
1628 private void resetIntent(final Bundle extras) {
1629 final Intent intent = new Intent(Intent.ACTION_VIEW);
1630 intent.putExtras(extras);
1631 setIntent(intent);
1632 }
1633
1634 private void resetIntent(
1635 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1636 final Intent intent = new Intent(Intent.ACTION_VIEW);
1637 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1638 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1639 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1640 } else {
1641 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1642 }
1643 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1644 intent.putExtra(
1645 EXTRA_LAST_ACTION,
1646 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1647 setIntent(intent);
1648 }
1649
1650 private static boolean emptyReference(final WeakReference<?> weakReference) {
1651 return weakReference == null || weakReference.get() == null;
1652 }
1653
1654 private enum Event {
1655 ON_BACKEND_CONNECTED,
1656 ON_NEW_INTENT
1657 }
1658}