1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import org.jetbrains.annotations.NotNull;
44import org.webrtc.RendererCommon;
45import org.webrtc.SurfaceViewRenderer;
46import org.webrtc.VideoTrack;
47
48import java.lang.ref.WeakReference;
49import java.util.Arrays;
50import java.util.Collections;
51import java.util.List;
52import java.util.Set;
53
54import eu.siacs.conversations.Config;
55import eu.siacs.conversations.R;
56import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
57import eu.siacs.conversations.entities.Account;
58import eu.siacs.conversations.entities.Contact;
59import eu.siacs.conversations.entities.Conversation;
60import eu.siacs.conversations.services.CallIntegration;
61import eu.siacs.conversations.services.CallIntegrationConnectionService;
62import eu.siacs.conversations.services.XmppConnectionService;
63import eu.siacs.conversations.ui.widget.DialpadView;
64import eu.siacs.conversations.ui.util.AvatarWorkerTask;
65import eu.siacs.conversations.ui.util.MainThreadExecutor;
66import eu.siacs.conversations.ui.util.Rationals;
67import eu.siacs.conversations.utils.PermissionUtils;
68import eu.siacs.conversations.utils.TimeFrameUtils;
69import eu.siacs.conversations.xmpp.Jid;
70import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
71import eu.siacs.conversations.xmpp.jingle.ContentAddition;
72import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
73import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
74import eu.siacs.conversations.xmpp.jingle.Media;
75import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
76import eu.siacs.conversations.xmpp.jingle.RtpCapability;
77import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
78
79import org.webrtc.RendererCommon;
80import org.webrtc.SurfaceViewRenderer;
81import org.webrtc.VideoTrack;
82
83import java.lang.ref.WeakReference;
84import java.util.Arrays;
85import java.util.Collections;
86import java.util.List;
87import java.util.Set;
88
89public class RtpSessionActivity extends XmppActivity
90 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
91 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
92
93 public static final String EXTRA_WITH = "with";
94 public static final String EXTRA_SESSION_ID = "session_id";
95 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
96 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
97 public static final String EXTRA_LAST_ACTION = "last_action";
98 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
99 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
100 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
101
102 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
103
104 public static final List<RtpEndUserState> END_CARD =
105 Arrays.asList(
106 RtpEndUserState.APPLICATION_ERROR,
107 RtpEndUserState.SECURITY_ERROR,
108 RtpEndUserState.DECLINED_OR_BUSY,
109 RtpEndUserState.CONTACT_OFFLINE,
110 RtpEndUserState.CONNECTIVITY_ERROR,
111 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
112 RtpEndUserState.RETRACTED);
113 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
114 Arrays.asList(
115 RtpEndUserState.APPLICATION_ERROR,
116 RtpEndUserState.CONNECTIVITY_ERROR,
117 RtpEndUserState.SECURITY_ERROR);
118 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
119 Arrays.asList(
120 RtpEndUserState.CONNECTING,
121 RtpEndUserState.CONNECTED,
122 RtpEndUserState.RECONNECTING,
123 RtpEndUserState.INCOMING_CONTENT_ADD);
124 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
125 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
126 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
127 Arrays.asList(
128 RtpEndUserState.ACCEPTING_CALL,
129 RtpEndUserState.CONNECTING,
130 RtpEndUserState.RECONNECTING);
131 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
132 new ImmutableList.Builder<RtpEndUserState>()
133 .add(RtpEndUserState.FINDING_DEVICE)
134 .add(RtpEndUserState.RINGING)
135 .add(RtpEndUserState.ACCEPTING_CALL)
136 .add(RtpEndUserState.CONNECTING)
137 .addAll(STATES_CONSIDERED_CONNECTED)
138 .build();
139 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
140 private static final int REQUEST_ACCEPT_CALL = 0x1111;
141 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
142 private static final int REQUEST_ADD_CONTENT = 0x1113;
143 private WeakReference<JingleRtpConnection> rtpConnectionReference;
144
145 private ActivityRtpSessionBinding binding;
146 private PowerManager.WakeLock mProximityWakeLock;
147
148 private final Handler mHandler = new Handler();
149 private final Runnable mTickExecutor =
150 new Runnable() {
151 @Override
152 public void run() {
153 updateCallDuration();
154 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
155 }
156 };
157
158 public static Set<Media> actionToMedia(final String action) {
159 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
160 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
161 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
162 return ImmutableSet.of(Media.AUDIO);
163 } else {
164 Log.w(
165 Config.LOGTAG,
166 "actionToMedia can not get media set from unknown action " + action);
167 return Collections.emptySet();
168 }
169 }
170
171 private static void addSink(
172 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
173 try {
174 videoTrack.addSink(surfaceViewRenderer);
175 } catch (final IllegalStateException e) {
176 Log.e(
177 Config.LOGTAG,
178 "possible race condition on trying to display video track. ignoring",
179 e);
180 }
181 }
182
183 @Override
184 public void onCreate(Bundle savedInstanceState) {
185 super.onCreate(savedInstanceState);
186 getWindow()
187 .addFlags(
188 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
189 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
190 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
191 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
192 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
193 setSupportActionBar(binding.toolbar);
194
195 binding.dialpad.setClickConsumer(tag -> {
196 final JingleRtpConnection connection =
197 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
198 if (connection != null) connection.applyDtmfTone(tag);
199 });
200
201 if (savedInstanceState != null) {
202 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
203 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
204 }
205
206 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
207 }
208
209 @Override
210 public boolean onCreateOptionsMenu(final Menu menu) {
211 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
212 final MenuItem help = menu.findItem(R.id.action_help);
213 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
214 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
215 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
216 help.setVisible(Config.HELP != null && isHelpButtonVisible());
217 gotoChat.setVisible(isSwitchToConversationVisible());
218 switchToVideo.setVisible(isSwitchToVideoVisible());
219 dialpad.setVisible(isAudioOnlyConversation());
220 return super.onCreateOptionsMenu(menu);
221 }
222
223 @Override
224 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
225 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
226 if (xmppConnectionService != null) {
227 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
228 return true;
229 }
230 }
231 }
232 return super.onKeyDown(keyCode, event);
233 }
234
235 private boolean isHelpButtonVisible() {
236 try {
237 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
238 } catch (IllegalStateException e) {
239 final Intent intent = getIntent();
240 final String state =
241 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
242 if (state != null) {
243 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
244 } else {
245 return false;
246 }
247 }
248 }
249
250 private boolean isSwitchToConversationVisible() {
251 final JingleRtpConnection connection =
252 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
253 return connection != null
254 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
255 }
256
257 private boolean isAudioOnlyConversation() {
258 final JingleRtpConnection connection =
259 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
260
261 return connection != null && !connection.getMedia().contains(Media.VIDEO);
262 }
263
264 private boolean isSwitchToVideoVisible() {
265 final JingleRtpConnection connection =
266 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
267 if (connection == null) {
268 return false;
269 }
270 return connection.isSwitchToVideoAvailable();
271 }
272
273 private void switchToConversation() {
274 final Contact contact = getWith();
275 final Conversation conversation =
276 xmppConnectionService.findOrCreateConversation(
277 contact.getAccount(), contact.getJid(), false, true);
278 switchToConversation(conversation);
279 }
280
281 private void toggleDialpadVisibility() {
282 if (binding.dialpad.getVisibility() == View.VISIBLE) {
283 binding.dialpad.setVisibility(View.GONE);
284 }
285 else {
286 binding.dialpad.setVisibility(View.VISIBLE);
287 }
288 }
289
290 public boolean onOptionsItemSelected(final MenuItem item) {
291 final var itemItem = item.getItemId();
292 if (itemItem == R.id.action_help) {
293 launchHelpInBrowser();
294 return true;
295 } else if (itemItem == R.id.action_goto_chat) {
296 switchToConversation();
297 return true;
298 } else if (itemItem == R.id.action_dialpad) {
299 toggleDialpadVisibility();
300 return true;
301 } else if (itemItem == R.id.action_switch_to_video) {
302 requestPermissionAndSwitchToVideo();
303 return true;
304 } else {
305 return super.onOptionsItemSelected(item);
306 }
307 }
308
309 private void launchHelpInBrowser() {
310 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
311 try {
312 startActivity(intent);
313 } catch (final ActivityNotFoundException e) {
314 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
315 .show();
316 }
317 }
318
319 private void endCall(View view) {
320 endCall();
321 }
322
323 private void endCall() {
324 if (this.rtpConnectionReference == null) {
325 retractSessionProposal();
326 finish();
327 } else {
328 requireRtpConnection().endCall();
329 }
330 }
331
332 private void retractSessionProposal() {
333 final Intent intent = getIntent();
334 final String action = intent.getAction();
335 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
336 final Account account = extractAccount(intent);
337 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
338 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
339 if (!Intent.ACTION_VIEW.equals(action)
340 || state == null
341 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
342 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
343 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
344 }
345 xmppConnectionService
346 .getJingleConnectionManager()
347 .retractSessionProposal(account, with.asBareJid());
348 }
349
350 private void rejectCall(View view) {
351 requireRtpConnection().rejectCall();
352 finish();
353 }
354
355 private void acceptCall(View view) {
356 requestPermissionsAndAcceptCall();
357 }
358
359 private void acceptContentAdd() {
360 try {
361 requireRtpConnection()
362 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
363 } catch (final IllegalStateException e) {
364 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
365 }
366 }
367
368 private void requestPermissionAndSwitchToVideo() {
369 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
370 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
371 switchToVideo();
372 }
373 }
374
375 private void switchToVideo() {
376 try {
377 requireRtpConnection().addMedia(Media.VIDEO);
378 } catch (final IllegalStateException e) {
379 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
380 }
381 }
382
383 private void acceptContentAdd(final ContentAddition contentAddition) {
384 if (contentAddition == null
385 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
386 Log.d(Config.LOGTAG, "ignore press on content-accept button");
387 return;
388 }
389 requestPermissionAndAcceptContentAdd(contentAddition);
390 }
391
392 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
393 final List<String> permissions = permissions(contentAddition.media());
394 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
395 try {
396 requireRtpConnection().acceptContentAdd(contentAddition.summary);
397 } catch (final IllegalStateException e) {
398 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
399 }
400 }
401 }
402
403 private void rejectContentAdd(final View view) {
404 requireRtpConnection().rejectContentAdd();
405 }
406
407 private void requestPermissionsAndAcceptCall() {
408 final List<String> permissions = permissions(getMedia());
409 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
410 putScreenInCallMode();
411 acceptCall();
412 }
413 }
414
415 private List<String> permissions(final Set<Media> media) {
416 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
417 if (media.contains(Media.VIDEO)) {
418 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
419 } else {
420 permissions.add(Manifest.permission.RECORD_AUDIO);
421 }
422 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
423 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
424 }
425 return permissions.build();
426 }
427
428 private void acceptCall() {
429 try {
430 requireRtpConnection().acceptCall();
431 } catch (final IllegalStateException e) {
432 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
433 }
434 }
435
436 private void putScreenInCallMode() {
437 putScreenInCallMode(requireRtpConnection().getMedia());
438 }
439
440 private void putScreenInCallMode(final Set<Media> media) {
441 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
442 if (Media.audioOnly(media)) {
443 final JingleRtpConnection rtpConnection =
444 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
445 final CallIntegration callIntegration =
446 rtpConnection == null ? null : rtpConnection.getCallIntegration();
447 if (callIntegration == null
448 || callIntegration.getSelectedAudioDevice()
449 == CallIntegration.AudioDevice.EARPIECE) {
450 acquireProximityWakeLock();
451 }
452 }
453 lockOrientation(media);
454 }
455
456 private void lockOrientation(final Set<Media> media) {
457 if (Media.audioOnly(media)) {
458 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
459 } else {
460 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
461 }
462 }
463
464 @SuppressLint("WakelockTimeout")
465 private void acquireProximityWakeLock() {
466 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
467 if (powerManager == null) {
468 Log.e(Config.LOGTAG, "power manager not available");
469 return;
470 }
471 if (isFinishing()) {
472 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
473 return;
474 }
475 if (this.mProximityWakeLock == null) {
476 this.mProximityWakeLock =
477 powerManager.newWakeLock(
478 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
479 }
480 if (!this.mProximityWakeLock.isHeld()) {
481 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
482 this.mProximityWakeLock.acquire();
483 }
484 }
485
486 private void releaseProximityWakeLock() {
487 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
488 Log.d(Config.LOGTAG, "releasing proximity wake lock");
489 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
490 this.mProximityWakeLock = null;
491 }
492 }
493
494 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
495 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
496 acquireProximityWakeLock();
497 } else {
498 releaseProximityWakeLock();
499 }
500 }
501
502 @Override
503 protected void refreshUiReal() {}
504
505 @Override
506 public void onNewIntent(final Intent intent) {
507 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
508 super.onNewIntent(intent);
509 if (intent == null) {
510 return;
511 }
512 setIntent(intent);
513 if (xmppConnectionService == null) {
514 Log.d(
515 Config.LOGTAG,
516 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
517 return;
518 }
519 initializeWithIntent(Event.ON_NEW_INTENT, intent);
520 }
521
522 @Override
523 protected void onBackendConnected() {
524 final var intent = getIntent();
525 if (intent == null) {
526 return;
527 }
528 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
529 }
530
531 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
532 final String action = intent.getAction();
533 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
534 final Account account = extractAccount(intent);
535 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
536 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
537 if (sessionId != null) {
538 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
539 return;
540 }
541 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
542 Log.d(Config.LOGTAG, "intent action was accept");
543 requestPermissionsAndAcceptCall();
544 resetIntent(intent.getExtras());
545 }
546 } else if (Intent.ACTION_VIEW.equals(action)) {
547 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
548 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
549 xmppConnectionService
550 .getJingleConnectionManager()
551 .getTerminalSessionState(with, proposedSessionId);
552 if (terminatedRtpSession != null) {
553 // termination (due to message error or 'busy' was faster than opening the activity
554 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
555 return;
556 }
557 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
558 final RtpEndUserState state =
559 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
560 if (state != null) {
561 Log.d(Config.LOGTAG, "restored last state from intent extra");
562 updateButtonConfiguration(state);
563 updateVerifiedShield(false);
564 updateStateDisplay(state);
565 updateIncomingCallScreen(state);
566 invalidateOptionsMenu();
567 }
568 setWith(account.getRoster().getContact(with), state);
569 if (xmppConnectionService
570 .getJingleConnectionManager()
571 .fireJingleRtpConnectionStateUpdates()) {
572 return;
573 }
574 if (END_CARD.contains(state)) {
575 return;
576 }
577 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
578 final Set<Media> media = actionToMedia(lastAction);
579 if (xmppConnectionService
580 .getJingleConnectionManager()
581 .hasMatchingProposal(account, with)) {
582 putScreenInCallMode(media);
583 return;
584 }
585 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
586 finish();
587 }
588 }
589
590 private void setWith(final RtpEndUserState state) {
591 setWith(getWith(), state);
592 }
593
594 private void setWith(final Contact contact, final RtpEndUserState state) {
595 binding.with.setText(contact.getDisplayName());
596 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
597 .contains(state)) {
598 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
599 binding.withJid.setVisibility(View.VISIBLE);
600 } else {
601 binding.withJid.setVisibility(View.GONE);
602 }
603 }
604
605 @Override
606 public void onRequestPermissionsResult(
607 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
608 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
609 final PermissionUtils.PermissionResult permissionResult =
610 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
611 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
612 if (requestCode == REQUEST_ACCEPT_CALL) {
613 acceptCall();
614 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
615 acceptContentAdd();
616 } else if (requestCode == REQUEST_ADD_CONTENT) {
617 switchToVideo();
618 }
619 } else {
620 @StringRes int res;
621 final String firstDenied =
622 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
623 if (firstDenied == null) {
624 return;
625 }
626 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
627 res = R.string.no_microphone_permission;
628 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
629 res = R.string.no_camera_permission;
630 } else {
631 throw new IllegalStateException("Invalid permission result request");
632 }
633 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
634 .show();
635 }
636 }
637
638 @Override
639 public void onStart() {
640 super.onStart();
641 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
642 this.binding.remoteVideo.setOnAspectRatioChanged(this);
643 }
644
645 @Override
646 public void onStop() {
647 mHandler.removeCallbacks(mTickExecutor);
648 binding.remoteVideo.release();
649 binding.remoteVideo.setOnAspectRatioChanged(null);
650 binding.localVideo.release();
651 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
652 final JingleRtpConnection jingleRtpConnection =
653 weakReference == null ? null : weakReference.get();
654 if (jingleRtpConnection != null) {
655 releaseVideoTracks(jingleRtpConnection);
656 }
657 releaseProximityWakeLock();
658 super.onStop();
659 }
660
661 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
662 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
663 if (remoteVideo.isPresent()) {
664 remoteVideo.get().removeSink(binding.remoteVideo);
665 }
666 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
667 if (localVideo.isPresent()) {
668 localVideo.get().removeSink(binding.localVideo);
669 }
670 }
671
672 @Override
673 public void onBackPressed() {
674 if (isConnected()) {
675 if (switchToPictureInPicture()) {
676 return;
677 }
678 } else {
679 endCall();
680 }
681 super.onBackPressed();
682 }
683
684 @Override
685 public void onUserLeaveHint() {
686 super.onUserLeaveHint();
687 if (switchToPictureInPicture()) {
688 return;
689 }
690 // TODO apparently this method is not getting called on Android 10 when using the task
691 // switcher
692 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
693 retractSessionProposal();
694 }
695 }
696
697 private boolean isConnected() {
698 final JingleRtpConnection connection =
699 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
700 final RtpEndUserState endUserState =
701 connection == null ? null : connection.getEndUserState();
702 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
703 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
704 }
705
706 private boolean switchToPictureInPicture() {
707 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
708 if (shouldBePictureInPicture()) {
709 startPictureInPicture();
710 return true;
711 }
712 }
713 return false;
714 }
715
716 @RequiresApi(api = Build.VERSION_CODES.O)
717 private void startPictureInPicture() {
718 try {
719 final Rational rational = this.binding.remoteVideo.getAspectRatio();
720 final Rational clippedRational = Rationals.clip(rational);
721 Log.d(
722 Config.LOGTAG,
723 "suggested rational " + rational + ". clipped to " + clippedRational);
724 enterPictureInPictureMode(
725 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
726 } catch (final IllegalStateException e) {
727 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
728 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
729 }
730 }
731
732 @Override
733 public void onAspectRatioChanged(final Rational rational) {
734 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
735 final Rational clippedRational = Rationals.clip(rational);
736 Log.d(
737 Config.LOGTAG,
738 "suggested rational after aspect ratio change "
739 + rational
740 + ". clipped to "
741 + clippedRational);
742 setPictureInPictureParams(
743 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
744 }
745 }
746
747 private boolean deviceSupportsPictureInPicture() {
748 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
749 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
750 } else {
751 return false;
752 }
753 }
754
755 private boolean shouldBePictureInPicture() {
756 try {
757 final JingleRtpConnection rtpConnection = requireRtpConnection();
758 return rtpConnection.getMedia().contains(Media.VIDEO)
759 && Arrays.asList(
760 RtpEndUserState.ACCEPTING_CALL,
761 RtpEndUserState.CONNECTING,
762 RtpEndUserState.CONNECTED)
763 .contains(rtpConnection.getEndUserState());
764 } catch (final IllegalStateException e) {
765 return false;
766 }
767 }
768
769 private boolean initializeActivityWithRunningRtpSession(
770 final Account account, Jid with, String sessionId) {
771 final WeakReference<JingleRtpConnection> reference =
772 xmppConnectionService
773 .getJingleConnectionManager()
774 .findJingleRtpConnection(account, with, sessionId);
775 if (reference == null || reference.get() == null) {
776 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
777 xmppConnectionService
778 .getJingleConnectionManager()
779 .getTerminalSessionState(with, sessionId);
780 if (terminatedRtpSession == null) {
781 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
782 finish();
783 return true;
784 }
785 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
786 return true;
787 }
788 this.rtpConnectionReference = reference;
789 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
790 final boolean verified = requireRtpConnection().isVerified();
791 if (currentState == RtpEndUserState.ENDED) {
792 finish();
793 return true;
794 }
795 final Set<Media> media = getMedia();
796 final ContentAddition contentAddition = getPendingContentAddition();
797 if (currentState == RtpEndUserState.INCOMING_CALL) {
798 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
799 }
800 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
801 requireRtpConnection().getState())) {
802 putScreenInCallMode();
803 }
804 setWith(currentState);
805 updateVideoViews(currentState);
806 updateStateDisplay(currentState, media, contentAddition);
807 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
808 updateButtonConfiguration(currentState, media, contentAddition);
809 updateIncomingCallScreen(currentState);
810 invalidateOptionsMenu();
811 return false;
812 }
813
814 private void initializeWithTerminatedSessionState(
815 final Account account,
816 final Jid with,
817 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
818 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
819 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
820 finish();
821 return;
822 }
823 final RtpEndUserState state = terminatedRtpSession.state;
824 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
825 updateButtonConfiguration(state);
826 updateStateDisplay(state);
827 updateIncomingCallScreen(state);
828 updateCallDuration();
829 updateVerifiedShield(false);
830 invalidateOptionsMenu();
831 setWith(account.getRoster().getContact(with), state);
832 }
833
834 private void reInitializeActivityWithRunningRtpSession(
835 final Account account, Jid with, String sessionId) {
836 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
837 resetIntent(account, with, sessionId);
838 }
839
840 private void resetIntent(final Account account, final Jid with, final String sessionId) {
841 final Intent intent = new Intent(Intent.ACTION_VIEW);
842 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
843 intent.putExtra(EXTRA_WITH, with.toEscapedString());
844 intent.putExtra(EXTRA_SESSION_ID, sessionId);
845 setIntent(intent);
846 }
847
848 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
849 surfaceViewRenderer.setVisibility(View.VISIBLE);
850 try {
851 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
852 } catch (final IllegalStateException ignored) {
853 // SurfaceViewRenderer was already initialized
854 } catch (final RuntimeException e) {
855 if (Throwables.getRootCause(e) instanceof GLException glException) {
856 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
857 }
858 }
859 surfaceViewRenderer.setEnableHardwareScaler(true);
860 }
861
862 private void updateStateDisplay(final RtpEndUserState state) {
863 updateStateDisplay(state, Collections.emptySet(), null);
864 }
865
866 private void updateStateDisplay(
867 final RtpEndUserState state,
868 final Set<Media> media,
869 final ContentAddition contentAddition) {
870 switch (state) {
871 case INCOMING_CALL -> {
872 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
873 if (media.contains(Media.VIDEO)) {
874 setTitle(R.string.rtp_state_incoming_video_call);
875 } else {
876 setTitle(R.string.rtp_state_incoming_call);
877 }
878 }
879 case INCOMING_CONTENT_ADD -> {
880 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
881 setTitle(R.string.rtp_state_content_add_video);
882 } else {
883 setTitle(R.string.rtp_state_content_add);
884 }
885 }
886 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
887 case CONNECTED -> setTitle(R.string.rtp_state_connected);
888 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
889 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
890 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
891 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
892 case RINGING -> setTitle(R.string.rtp_state_ringing);
893 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
894 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
895 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
896 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
897 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
898 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
899 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
900 case ENDED -> throw new IllegalStateException(
901 "Activity should have called finishAndReleaseWakeLock();");
902 default -> throw new IllegalStateException(
903 String.format("State %s has not been handled in UI", state));
904 }
905 }
906
907 private void updateVerifiedShield(final boolean verified) {
908 if (isPictureInPicture()) {
909 this.binding.verified.setVisibility(View.GONE);
910 return;
911 }
912 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
913 }
914
915 private void updateIncomingCallScreen(final RtpEndUserState state) {
916 updateIncomingCallScreen(state, null);
917 }
918
919 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
920 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
921 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
922 if (show) {
923 binding.contactPhoto.setVisibility(View.VISIBLE);
924 if (contact == null) {
925 AvatarWorkerTask.loadAvatar(
926 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
927 } else {
928 AvatarWorkerTask.loadAvatar(
929 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
930 }
931 } else {
932 binding.contactPhoto.setVisibility(View.GONE);
933 }
934 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
935 binding.usingAccount.setVisibility(View.VISIBLE);
936 binding.usingAccount.setText(
937 getString(
938 R.string.using_account,
939 account.getJid().asBareJid().toEscapedString()));
940 } else {
941 binding.usingAccount.setVisibility(View.GONE);
942 binding.contactPhoto.setVisibility(View.GONE);
943 }
944 }
945
946 private Set<Media> getMedia() {
947 return requireRtpConnection().getMedia();
948 }
949
950 public ContentAddition getPendingContentAddition() {
951 return requireRtpConnection().getPendingContentAddition();
952 }
953
954 private void updateButtonConfiguration(final RtpEndUserState state) {
955 updateButtonConfiguration(state, Collections.emptySet(), null);
956 }
957
958 @SuppressLint("RestrictedApi")
959 private void updateButtonConfiguration(
960 final RtpEndUserState state,
961 final Set<Media> media,
962 final ContentAddition contentAddition) {
963 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
964 this.binding.rejectCall.setVisibility(View.INVISIBLE);
965 this.binding.endCall.setVisibility(View.INVISIBLE);
966 this.binding.acceptCall.setVisibility(View.INVISIBLE);
967 } else if (state == RtpEndUserState.INCOMING_CALL) {
968 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
969 this.binding.rejectCall.setOnClickListener(this::rejectCall);
970 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
971 this.binding.rejectCall.setVisibility(View.VISIBLE);
972 this.binding.endCall.setVisibility(View.INVISIBLE);
973 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
974 this.binding.acceptCall.setOnClickListener(this::acceptCall);
975 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
976 this.binding.acceptCall.setVisibility(View.VISIBLE);
977 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
978 this.binding.rejectCall.setContentDescription(
979 getString(R.string.reject_switch_to_video));
980 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
981 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
982 this.binding.rejectCall.setVisibility(View.VISIBLE);
983 this.binding.endCall.setVisibility(View.INVISIBLE);
984 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
985 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
986 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
987 this.binding.acceptCall.setVisibility(View.VISIBLE);
988 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
989 .contains(state)) {
990 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
991 this.binding.rejectCall.setOnClickListener(this::exit);
992 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
993 this.binding.rejectCall.setVisibility(View.VISIBLE);
994 this.binding.endCall.setVisibility(View.INVISIBLE);
995 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
996 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
997 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
998 this.binding.acceptCall.setVisibility(View.VISIBLE);
999 } else if (asList(
1000 RtpEndUserState.CONNECTIVITY_ERROR,
1001 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1002 RtpEndUserState.APPLICATION_ERROR,
1003 RtpEndUserState.RETRACTED,
1004 RtpEndUserState.SECURITY_ERROR)
1005 .contains(state)) {
1006 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1007 this.binding.rejectCall.setOnClickListener(this::exit);
1008 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1009 this.binding.rejectCall.setVisibility(View.VISIBLE);
1010 this.binding.endCall.setVisibility(View.INVISIBLE);
1011 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1012 this.binding.acceptCall.setOnClickListener(this::retry);
1013 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1014 this.binding.acceptCall.setVisibility(View.VISIBLE);
1015 } else {
1016 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1017 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1018 this.binding.endCall.setOnClickListener(this::endCall);
1019 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1020 this.binding.endCall.setVisibility(View.VISIBLE);
1021 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1022 }
1023 updateInCallButtonConfiguration(state, media);
1024 }
1025
1026 private boolean isPictureInPicture() {
1027 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1028 return isInPictureInPictureMode();
1029 } else {
1030 return false;
1031 }
1032 }
1033
1034 private void updateInCallButtonConfiguration() {
1035 updateInCallButtonConfiguration(
1036 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1037 }
1038
1039 @SuppressLint("RestrictedApi")
1040 private void updateInCallButtonConfiguration(
1041 final RtpEndUserState state, final Set<Media> media) {
1042 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1043 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1044 if (media.contains(Media.VIDEO)) {
1045 final JingleRtpConnection rtpConnection = requireRtpConnection();
1046 updateInCallButtonConfigurationVideo(
1047 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1048 } else {
1049 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1050 updateInCallButtonConfigurationSpeaker(
1051 callIntegration.getSelectedAudioDevice(),
1052 callIntegration.getAudioDevices().size());
1053 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1054 }
1055 if (media.contains(Media.AUDIO)) {
1056 updateInCallButtonConfigurationMicrophone(
1057 requireRtpConnection().isMicrophoneEnabled());
1058 } else {
1059 this.binding.inCallActionLeft.setVisibility(View.GONE);
1060 }
1061 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1062 && !isPictureInPicture()
1063 && Media.audioOnly(media)) {
1064 final CallIntegration callIntegration;
1065 try {
1066 callIntegration = requireCallIntegration();
1067 } catch (final IllegalStateException e) {
1068 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1069 return;
1070 }
1071 updateInCallButtonConfigurationSpeaker(
1072 callIntegration.getSelectedAudioDevice(),
1073 callIntegration.getAudioDevices().size());
1074 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1075 } else {
1076 this.binding.inCallActionLeft.setVisibility(View.GONE);
1077 this.binding.inCallActionRight.setVisibility(View.GONE);
1078 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1079 }
1080 }
1081
1082 @SuppressLint("RestrictedApi")
1083 private void updateInCallButtonConfigurationSpeaker(
1084 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1085 switch (selectedAudioDevice) {
1086 case EARPIECE -> {
1087 this.binding.inCallActionRight.setImageResource(
1088 R.drawable.ic_volume_off_24dp);
1089 if (numberOfChoices >= 2) {
1090 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1091 } else {
1092 this.binding.inCallActionRight.setOnClickListener(null);
1093 this.binding.inCallActionRight.setClickable(false);
1094 }
1095 }
1096 case WIRED_HEADSET -> {
1097 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1098 this.binding.inCallActionRight.setOnClickListener(null);
1099 this.binding.inCallActionRight.setClickable(false);
1100 }
1101 case SPEAKER_PHONE -> {
1102 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1103 if (numberOfChoices >= 2) {
1104 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1105 } else {
1106 this.binding.inCallActionRight.setOnClickListener(null);
1107 this.binding.inCallActionRight.setClickable(false);
1108 }
1109 }
1110 case BLUETOOTH -> {
1111 this.binding.inCallActionRight.setImageResource(
1112 R.drawable.ic_bluetooth_audio_24dp);
1113 this.binding.inCallActionRight.setOnClickListener(null);
1114 this.binding.inCallActionRight.setClickable(false);
1115 }
1116 }
1117 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1118 }
1119
1120 @SuppressLint("RestrictedApi")
1121 private void updateInCallButtonConfigurationVideo(
1122 final boolean videoEnabled, final boolean isCameraSwitchable) {
1123 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1124 if (isCameraSwitchable) {
1125 this.binding.inCallActionFarRight.setImageResource(
1126 R.drawable.ic_flip_camera_android_24dp);
1127 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1128 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1129 } else {
1130 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1131 }
1132 if (videoEnabled) {
1133 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1134 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1135 } else {
1136 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1137 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1138 }
1139 }
1140
1141 private void switchCamera(final View view) {
1142 Futures.addCallback(
1143 requireRtpConnection().switchCamera(),
1144 new FutureCallback<>() {
1145 @Override
1146 public void onSuccess(@Nullable Boolean isFrontCamera) {
1147 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1148 }
1149
1150 @Override
1151 public void onFailure(@NonNull final Throwable throwable) {
1152 Log.d(
1153 Config.LOGTAG,
1154 "could not switch camera",
1155 Throwables.getRootCause(throwable));
1156 Toast.makeText(
1157 RtpSessionActivity.this,
1158 R.string.could_not_switch_camera,
1159 Toast.LENGTH_LONG)
1160 .show();
1161 }
1162 },
1163 MainThreadExecutor.getInstance());
1164 }
1165
1166 private void enableVideo(View view) {
1167 try {
1168 requireRtpConnection().setVideoEnabled(true);
1169 } catch (final IllegalStateException e) {
1170 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1171 return;
1172 }
1173 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1174 }
1175
1176 private void disableVideo(View view) {
1177 final JingleRtpConnection rtpConnection = requireRtpConnection();
1178 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1179 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1180 rtpConnection.retractContentAdd();
1181 return;
1182 }
1183 requireRtpConnection().setVideoEnabled(false);
1184 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1185 }
1186
1187 @SuppressLint("RestrictedApi")
1188 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1189 if (microphoneEnabled) {
1190 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1191 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1192 } else {
1193 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1194 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1195 }
1196 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1197 }
1198
1199 private void updateCallDuration() {
1200 final JingleRtpConnection connection =
1201 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1202 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1203 this.binding.duration.setVisibility(View.GONE);
1204 return;
1205 }
1206 if (connection.zeroDuration()) {
1207 this.binding.duration.setVisibility(View.GONE);
1208 } else {
1209 this.binding.duration.setText(
1210 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1211 this.binding.duration.setVisibility(View.VISIBLE);
1212 }
1213 }
1214
1215 private void updateVideoViews(final RtpEndUserState state) {
1216 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1217 binding.localVideo.setVisibility(View.GONE);
1218 binding.localVideo.release();
1219 binding.remoteVideoWrapper.setVisibility(View.GONE);
1220 binding.remoteVideo.release();
1221 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1222 if (isPictureInPicture()) {
1223 binding.appBarLayout.setVisibility(View.GONE);
1224 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1225 if (Arrays.asList(
1226 RtpEndUserState.APPLICATION_ERROR,
1227 RtpEndUserState.CONNECTIVITY_ERROR,
1228 RtpEndUserState.SECURITY_ERROR)
1229 .contains(state)) {
1230 binding.pipWarning.setVisibility(View.VISIBLE);
1231 binding.pipWaiting.setVisibility(View.GONE);
1232 } else {
1233 binding.pipWarning.setVisibility(View.GONE);
1234 binding.pipWaiting.setVisibility(View.GONE);
1235 }
1236 } else {
1237 binding.appBarLayout.setVisibility(View.VISIBLE);
1238 binding.pipPlaceholder.setVisibility(View.GONE);
1239 }
1240 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1241 return;
1242 }
1243 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1244 binding.localVideo.setVisibility(View.GONE);
1245 binding.remoteVideoWrapper.setVisibility(View.GONE);
1246 binding.appBarLayout.setVisibility(View.GONE);
1247 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1248 binding.pipWarning.setVisibility(View.GONE);
1249 binding.pipWaiting.setVisibility(View.VISIBLE);
1250 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1251 return;
1252 }
1253 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1254 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1255 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1256 // paint local view over remote view
1257 binding.localVideo.setZOrderMediaOverlay(true);
1258 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1259 addSink(localVideoTrack.get(), binding.localVideo);
1260 } else {
1261 binding.localVideo.setVisibility(View.GONE);
1262 }
1263 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1264 if (remoteVideoTrack.isPresent()) {
1265 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1266 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1267 binding.remoteVideo.setScalingType(
1268 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1269 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1270 if (state == RtpEndUserState.CONNECTED) {
1271 binding.appBarLayout.setVisibility(View.GONE);
1272 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1273 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1274 } else {
1275 binding.appBarLayout.setVisibility(View.VISIBLE);
1276 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1277 binding.remoteVideoWrapper.setVisibility(View.GONE);
1278 }
1279 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1280 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1281 } else {
1282 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1283 }
1284 } else {
1285 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1286 binding.remoteVideoWrapper.setVisibility(View.GONE);
1287 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1288 }
1289 }
1290
1291 private Optional<VideoTrack> getLocalVideoTrack() {
1292 final JingleRtpConnection connection =
1293 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1294 if (connection == null) {
1295 return Optional.absent();
1296 }
1297 return connection.getLocalVideoTrack();
1298 }
1299
1300 private Optional<VideoTrack> getRemoteVideoTrack() {
1301 final JingleRtpConnection connection =
1302 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1303 if (connection == null) {
1304 return Optional.absent();
1305 }
1306 return connection.getRemoteVideoTrack();
1307 }
1308
1309 private void disableMicrophone(View view) {
1310 final JingleRtpConnection rtpConnection = requireRtpConnection();
1311 if (rtpConnection.setMicrophoneEnabled(false)) {
1312 updateInCallButtonConfiguration();
1313 }
1314 }
1315
1316 private void enableMicrophone(View view) {
1317 final JingleRtpConnection rtpConnection = requireRtpConnection();
1318 if (rtpConnection.setMicrophoneEnabled(true)) {
1319 updateInCallButtonConfiguration();
1320 }
1321 }
1322
1323 private void switchToEarpiece(final View view) {
1324 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1325 acquireProximityWakeLock();
1326 }
1327
1328 private void switchToSpeaker(final View view) {
1329 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1330 releaseProximityWakeLock();
1331 }
1332
1333 private void retry(final View view) {
1334 final Intent intent = getIntent();
1335 final Account account = extractAccount(intent);
1336 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1337 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1338 final String action = intent.getAction();
1339 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1340 this.rtpConnectionReference = null;
1341 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1342 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1343 }
1344
1345 private void exit(final View view) {
1346 finish();
1347 }
1348
1349 private void recordVoiceMail(final View view) {
1350 final Intent intent = getIntent();
1351 final Account account = extractAccount(intent);
1352 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1353 final Conversation conversation =
1354 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1355 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1356 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1357 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1358 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1359 launchIntent.putExtra(
1360 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1361 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1362 startActivity(launchIntent);
1363 finish();
1364 }
1365
1366 private Contact getWith() {
1367 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1368 final Account account = id.account;
1369 return account.getRoster().getContact(id.with);
1370 }
1371
1372 private JingleRtpConnection requireRtpConnection() {
1373 final JingleRtpConnection connection =
1374 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1375 if (connection == null) {
1376 throw new IllegalStateException("No RTP connection found");
1377 }
1378 return connection;
1379 }
1380
1381 private CallIntegration requireCallIntegration() {
1382 return requireOngoingRtpSession().getCallIntegration();
1383 }
1384
1385 private OngoingRtpSession requireOngoingRtpSession() {
1386 final JingleRtpConnection connection =
1387 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1388 if (connection != null) {
1389 return connection;
1390 }
1391 final Intent currentIntent = getIntent();
1392 final String withExtra =
1393 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1394 final var account = extractAccount(currentIntent);
1395 if (withExtra == null) {
1396 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1397 }
1398 final var matching =
1399 xmppConnectionService
1400 .getJingleConnectionManager()
1401 .matchingProposal(account, Jid.of(withExtra));
1402 if (matching.isPresent()) {
1403 return matching.get();
1404 }
1405 throw new IllegalStateException("No matching session proposal");
1406 }
1407
1408 @Override
1409 public void onJingleRtpConnectionUpdate(
1410 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1411 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1412 if (END_CARD.contains(state)) {
1413 Log.d(Config.LOGTAG, "end card reached");
1414 releaseProximityWakeLock();
1415 runOnUiThread(
1416 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1417 }
1418 if (with.isBareJid()) {
1419 updateRtpSessionProposalState(account, with, state);
1420 return;
1421 }
1422 if (emptyReference(this.rtpConnectionReference)) {
1423 if (END_CARD.contains(state)) {
1424 Log.d(Config.LOGTAG, "not reinitializing session");
1425 return;
1426 }
1427 // this happens when going from proposed session to actual session
1428 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1429 return;
1430 }
1431 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1432 final boolean verified = requireRtpConnection().isVerified();
1433 final Set<Media> media = getMedia();
1434 lockOrientation(media);
1435 final ContentAddition contentAddition = getPendingContentAddition();
1436 final Contact contact = getWith();
1437 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1438 if (state == RtpEndUserState.ENDED) {
1439 finish();
1440 return;
1441 }
1442 runOnUiThread(
1443 () -> {
1444 updateStateDisplay(state, media, contentAddition);
1445 updateVerifiedShield(
1446 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1447 updateButtonConfiguration(state, media, contentAddition);
1448 updateVideoViews(state);
1449 updateIncomingCallScreen(state, contact);
1450 invalidateOptionsMenu();
1451 });
1452 if (END_CARD.contains(state)) {
1453 final JingleRtpConnection rtpConnection = requireRtpConnection();
1454 resetIntent(account, with, state, rtpConnection.getMedia());
1455 releaseVideoTracks(rtpConnection);
1456 this.rtpConnectionReference = null;
1457 }
1458 } else {
1459 Log.d(Config.LOGTAG, "received update for other rtp session");
1460 }
1461 }
1462
1463 @Override
1464 public void onAudioDeviceChanged(
1465 final CallIntegration.AudioDevice selectedAudioDevice,
1466 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1467 Log.d(
1468 Config.LOGTAG,
1469 "onAudioDeviceChanged in activity: selected:"
1470 + selectedAudioDevice
1471 + ", available:"
1472 + availableAudioDevices);
1473 try {
1474 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1475 final RtpEndUserState endUserState;
1476 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1477 endUserState = jingleRtpConnection.getEndUserState();
1478 } else {
1479 // for session proposals all end user states are functionally the same
1480 endUserState = RtpEndUserState.RINGING;
1481 }
1482 final Set<Media> media = ongoingRtpSession.getMedia();
1483 if (END_CARD.contains(endUserState)) {
1484 Log.d(
1485 Config.LOGTAG,
1486 "onAudioDeviceChanged() nothing to do because end card has been reached");
1487 } else {
1488 if (Media.audioOnly(media)
1489 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1490 final CallIntegration callIntegration = requireCallIntegration();
1491 updateInCallButtonConfigurationSpeaker(
1492 callIntegration.getSelectedAudioDevice(),
1493 callIntegration.getAudioDevices().size());
1494 }
1495 Log.d(
1496 Config.LOGTAG,
1497 "put proximity wake lock into proper state after device update");
1498 putProximityWakeLockInProperState(selectedAudioDevice);
1499 }
1500 } catch (final IllegalStateException e) {
1501 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1502 }
1503 }
1504
1505 @Override
1506 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1507 super.onSaveInstanceState(outState);
1508 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1509 }
1510
1511 private void updateRtpSessionProposalState(
1512 final Account account, final Jid with, final RtpEndUserState state) {
1513 final Intent currentIntent = getIntent();
1514 final String withExtra =
1515 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1516 if (withExtra == null) {
1517 return;
1518 }
1519 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1520 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1521 runOnUiThread(
1522 () -> {
1523 updateVerifiedShield(false);
1524 updateStateDisplay(state);
1525 updateButtonConfiguration(state, media, null);
1526 updateIncomingCallScreen(state);
1527 invalidateOptionsMenu();
1528 });
1529 resetIntent(account, with, state, media);
1530 }
1531 }
1532
1533 private void resetIntent(final Bundle extras) {
1534 final Intent intent = new Intent(Intent.ACTION_VIEW);
1535 intent.putExtras(extras);
1536 setIntent(intent);
1537 }
1538
1539 private void resetIntent(
1540 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1541 final Intent intent = new Intent(Intent.ACTION_VIEW);
1542 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1543 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1544 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1545 } else {
1546 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1547 }
1548 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1549 intent.putExtra(
1550 EXTRA_LAST_ACTION,
1551 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1552 setIntent(intent);
1553 }
1554
1555 private static boolean emptyReference(final WeakReference<?> weakReference) {
1556 return weakReference == null || weakReference.get() == null;
1557 }
1558
1559 private enum Event {
1560 ON_BACKEND_CONNECTED,
1561 ON_NEW_INTENT
1562 }
1563}