1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import eu.siacs.conversations.Config;
44import eu.siacs.conversations.R;
45import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
46import eu.siacs.conversations.entities.Account;
47import eu.siacs.conversations.entities.Contact;
48import eu.siacs.conversations.entities.Conversation;
49import eu.siacs.conversations.services.CallIntegration;
50import eu.siacs.conversations.services.CallIntegrationConnectionService;
51import eu.siacs.conversations.services.XmppConnectionService;
52import eu.siacs.conversations.ui.util.AvatarWorkerTask;
53import eu.siacs.conversations.ui.util.MainThreadExecutor;
54import eu.siacs.conversations.ui.util.Rationals;
55import eu.siacs.conversations.utils.PermissionUtils;
56import eu.siacs.conversations.utils.TimeFrameUtils;
57import eu.siacs.conversations.xmpp.Jid;
58import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
59import eu.siacs.conversations.xmpp.jingle.ContentAddition;
60import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
61import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
62import eu.siacs.conversations.xmpp.jingle.Media;
63import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
64import eu.siacs.conversations.xmpp.jingle.RtpCapability;
65import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
66
67import org.webrtc.RendererCommon;
68import org.webrtc.SurfaceViewRenderer;
69import org.webrtc.VideoTrack;
70
71import java.lang.ref.WeakReference;
72import java.util.Arrays;
73import java.util.Collections;
74import java.util.List;
75import java.util.Set;
76
77public class RtpSessionActivity extends XmppActivity
78 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
79 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
80
81 public static final String EXTRA_WITH = "with";
82 public static final String EXTRA_SESSION_ID = "session_id";
83 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
84 public static final String EXTRA_LAST_ACTION = "last_action";
85 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
86 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
87 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
88
89 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
90
91 private static final List<RtpEndUserState> END_CARD =
92 Arrays.asList(
93 RtpEndUserState.APPLICATION_ERROR,
94 RtpEndUserState.SECURITY_ERROR,
95 RtpEndUserState.DECLINED_OR_BUSY,
96 RtpEndUserState.CONTACT_OFFLINE,
97 RtpEndUserState.CONNECTIVITY_ERROR,
98 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
99 RtpEndUserState.RETRACTED);
100 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
101 Arrays.asList(
102 RtpEndUserState.APPLICATION_ERROR,
103 RtpEndUserState.CONNECTIVITY_ERROR,
104 RtpEndUserState.SECURITY_ERROR);
105 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
106 Arrays.asList(
107 RtpEndUserState.CONNECTING,
108 RtpEndUserState.CONNECTED,
109 RtpEndUserState.RECONNECTING,
110 RtpEndUserState.INCOMING_CONTENT_ADD);
111 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
112 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
113 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
114 Arrays.asList(
115 RtpEndUserState.ACCEPTING_CALL,
116 RtpEndUserState.CONNECTING,
117 RtpEndUserState.RECONNECTING);
118 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
119 new ImmutableList.Builder<RtpEndUserState>()
120 .add(RtpEndUserState.FINDING_DEVICE)
121 .add(RtpEndUserState.RINGING)
122 .add(RtpEndUserState.ACCEPTING_CALL)
123 .add(RtpEndUserState.CONNECTING)
124 .addAll(STATES_CONSIDERED_CONNECTED)
125 .build();
126 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
127 private static final int REQUEST_ACCEPT_CALL = 0x1111;
128 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
129 private static final int REQUEST_ADD_CONTENT = 0x1113;
130 private WeakReference<JingleRtpConnection> rtpConnectionReference;
131
132 private ActivityRtpSessionBinding binding;
133 private PowerManager.WakeLock mProximityWakeLock;
134
135 private final Handler mHandler = new Handler();
136 private final Runnable mTickExecutor =
137 new Runnable() {
138 @Override
139 public void run() {
140 updateCallDuration();
141 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
142 }
143 };
144
145 public static Set<Media> actionToMedia(final String action) {
146 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
147 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
148 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
149 return ImmutableSet.of(Media.AUDIO);
150 } else {
151 Log.w(
152 Config.LOGTAG,
153 "actionToMedia can not get media set from unknown action " + action);
154 return Collections.emptySet();
155 }
156 }
157
158 private static void addSink(
159 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
160 try {
161 videoTrack.addSink(surfaceViewRenderer);
162 } catch (final IllegalStateException e) {
163 Log.e(
164 Config.LOGTAG,
165 "possible race condition on trying to display video track. ignoring",
166 e);
167 }
168 }
169
170 @Override
171 public void onCreate(Bundle savedInstanceState) {
172 super.onCreate(savedInstanceState);
173 getWindow()
174 .addFlags(
175 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
176 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
177 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
178 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
179 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
180 setSupportActionBar(binding.toolbar);
181 }
182
183 @Override
184 public boolean onCreateOptionsMenu(final Menu menu) {
185 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
186 final MenuItem help = menu.findItem(R.id.action_help);
187 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
188 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
189 help.setVisible(Config.HELP != null && isHelpButtonVisible());
190 gotoChat.setVisible(isSwitchToConversationVisible());
191 switchToVideo.setVisible(isSwitchToVideoVisible());
192 return super.onCreateOptionsMenu(menu);
193 }
194
195 @Override
196 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
197 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
198 if (xmppConnectionService != null) {
199 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
200 return true;
201 }
202 }
203 }
204 return super.onKeyDown(keyCode, event);
205 }
206
207 private boolean isHelpButtonVisible() {
208 try {
209 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
210 } catch (IllegalStateException e) {
211 final Intent intent = getIntent();
212 final String state =
213 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
214 if (state != null) {
215 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
216 } else {
217 return false;
218 }
219 }
220 }
221
222 private boolean isSwitchToConversationVisible() {
223 final JingleRtpConnection connection =
224 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
225 return connection != null
226 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
227 }
228
229 private boolean isSwitchToVideoVisible() {
230 final JingleRtpConnection connection =
231 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
232 if (connection == null) {
233 return false;
234 }
235 return connection.isSwitchToVideoAvailable();
236 }
237
238 private void switchToConversation() {
239 final Contact contact = getWith();
240 final Conversation conversation =
241 xmppConnectionService.findOrCreateConversation(
242 contact.getAccount(), contact.getJid(), false, true);
243 switchToConversation(conversation);
244 }
245
246 public boolean onOptionsItemSelected(final MenuItem item) {
247 final var itemItem = item.getItemId();
248 if (itemItem == R.id.action_help) {
249 launchHelpInBrowser();
250 return true;
251 } else if (itemItem == R.id.action_goto_chat) {
252 switchToConversation();
253 return true;
254 } else if (itemItem == R.id.action_switch_to_video) {
255 requestPermissionAndSwitchToVideo();
256 return true;
257 } else {
258 return super.onOptionsItemSelected(item);
259 }
260 }
261
262 private void launchHelpInBrowser() {
263 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
264 try {
265 startActivity(intent);
266 } catch (final ActivityNotFoundException e) {
267 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
268 .show();
269 }
270 }
271
272 private void endCall(View view) {
273 endCall();
274 }
275
276 private void endCall() {
277 if (this.rtpConnectionReference == null) {
278 retractSessionProposal();
279 finish();
280 } else {
281 requireRtpConnection().endCall();
282 }
283 }
284
285 private void retractSessionProposal() {
286 final Intent intent = getIntent();
287 final String action = intent.getAction();
288 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
289 final Account account = extractAccount(intent);
290 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
291 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
292 if (!Intent.ACTION_VIEW.equals(action)
293 || state == null
294 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
295 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
296 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
297 }
298 xmppConnectionService
299 .getJingleConnectionManager()
300 .retractSessionProposal(account, with.asBareJid());
301 }
302
303 private void rejectCall(View view) {
304 requireRtpConnection().rejectCall();
305 finish();
306 }
307
308 private void acceptCall(View view) {
309 requestPermissionsAndAcceptCall();
310 }
311
312 private void acceptContentAdd() {
313 try {
314 requireRtpConnection()
315 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
316 } catch (final IllegalStateException e) {
317 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
318 }
319 }
320
321 private void requestPermissionAndSwitchToVideo() {
322 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
323 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
324 switchToVideo();
325 }
326 }
327
328 private void switchToVideo() {
329 try {
330 requireRtpConnection().addMedia(Media.VIDEO);
331 } catch (final IllegalStateException e) {
332 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
333 }
334 }
335
336 private void acceptContentAdd(final ContentAddition contentAddition) {
337 if (contentAddition == null
338 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
339 Log.d(Config.LOGTAG, "ignore press on content-accept button");
340 return;
341 }
342 requestPermissionAndAcceptContentAdd(contentAddition);
343 }
344
345 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
346 final List<String> permissions = permissions(contentAddition.media());
347 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
348 try {
349 requireRtpConnection().acceptContentAdd(contentAddition.summary);
350 } catch (final IllegalStateException e) {
351 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
352 }
353 }
354 }
355
356 private void rejectContentAdd(final View view) {
357 requireRtpConnection().rejectContentAdd();
358 }
359
360 private void requestPermissionsAndAcceptCall() {
361 final List<String> permissions = permissions(getMedia());
362 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
363 putScreenInCallMode();
364 acceptCall();
365 }
366 }
367
368 private List<String> permissions(final Set<Media> media) {
369 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
370 if (media.contains(Media.VIDEO)) {
371 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
372 } else {
373 permissions.add(Manifest.permission.RECORD_AUDIO);
374 }
375 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
376 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
377 }
378 return permissions.build();
379 }
380
381 private void acceptCall() {
382 try {
383 requireRtpConnection().acceptCall();
384 } catch (final IllegalStateException e) {
385 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
386 }
387 }
388
389
390 private void putScreenInCallMode() {
391 putScreenInCallMode(requireRtpConnection().getMedia());
392 }
393
394 private void putScreenInCallMode(final Set<Media> media) {
395 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
396 if (Media.audioOnly(media)) {
397 final JingleRtpConnection rtpConnection =
398 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
399 final CallIntegration callIntegration =
400 rtpConnection == null ? null : rtpConnection.getCallIntegration();
401 if (callIntegration == null
402 || callIntegration.getSelectedAudioDevice()
403 == CallIntegration.AudioDevice.EARPIECE) {
404 acquireProximityWakeLock();
405 }
406 }
407 lockOrientation(media);
408 }
409
410 private void lockOrientation(final Set<Media> media) {
411 if (Media.audioOnly(media)) {
412 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
413 } else {
414 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
415 }
416 }
417
418 @SuppressLint("WakelockTimeout")
419 private void acquireProximityWakeLock() {
420 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
421 if (powerManager == null) {
422 Log.e(Config.LOGTAG, "power manager not available");
423 return;
424 }
425 if (isFinishing()) {
426 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
427 return;
428 }
429 if (this.mProximityWakeLock == null) {
430 this.mProximityWakeLock =
431 powerManager.newWakeLock(
432 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
433 }
434 if (!this.mProximityWakeLock.isHeld()) {
435 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
436 this.mProximityWakeLock.acquire();
437 }
438 }
439
440 private void releaseProximityWakeLock() {
441 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
442 Log.d(Config.LOGTAG, "releasing proximity wake lock");
443 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
444 this.mProximityWakeLock = null;
445 }
446 }
447
448 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
449 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
450 acquireProximityWakeLock();
451 } else {
452 releaseProximityWakeLock();
453 }
454 }
455
456 @Override
457 protected void refreshUiReal() {}
458
459 @Override
460 public void onNewIntent(final Intent intent) {
461 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
462 super.onNewIntent(intent);
463 setIntent(intent);
464 if (xmppConnectionService == null) {
465 Log.d(
466 Config.LOGTAG,
467 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
468 return;
469 }
470 final Account account = extractAccount(intent);
471 final String action = intent.getAction();
472 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
473 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
474 if (sessionId != null) {
475 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
476 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
477 return;
478 }
479 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
480 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
481 requestPermissionsAndAcceptCall();
482 resetIntent(intent.getExtras());
483 }
484 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
485 proposeJingleRtpSession(account, with, actionToMedia(action));
486 setWith(account.getRoster().getContact(with), null);
487 } else {
488 throw new IllegalStateException("received onNewIntent without sessionId");
489 }
490 }
491
492 @Override
493 void onBackendConnected() {
494 final Intent intent = getIntent();
495 final String action = intent.getAction();
496 final Account account = extractAccount(intent);
497 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
498 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
499 if (sessionId != null) {
500 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
501 return;
502 }
503 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
504 Log.d(Config.LOGTAG, "intent action was accept");
505 requestPermissionsAndAcceptCall();
506 resetIntent(intent.getExtras());
507 }
508 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
509 proposeJingleRtpSession(account, with, actionToMedia(action));
510 setWith(account.getRoster().getContact(with), null);
511 } else if (Intent.ACTION_VIEW.equals(action)) {
512 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
513 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
514 final Set<Media> media = actionToMedia(lastAction);
515 final RtpEndUserState state =
516 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
517 if (state != null) {
518 Log.d(Config.LOGTAG, "restored last state from intent extra");
519 updateButtonConfiguration(state);
520 updateVerifiedShield(false);
521 updateStateDisplay(state);
522 updateIncomingCallScreen(state);
523 invalidateOptionsMenu();
524 }
525 setWith(account.getRoster().getContact(with), state);
526 if (xmppConnectionService
527 .getJingleConnectionManager()
528 .fireJingleRtpConnectionStateUpdates()) {
529 return;
530 }
531 if (END_CARD.contains(state)) {
532 return;
533 }
534 if (xmppConnectionService
535 .getJingleConnectionManager()
536 .hasMatchingProposal(account, with)) {
537 putScreenInCallMode(media);
538 return;
539 }
540 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
541 finish();
542 }
543 }
544
545 private void setWidth(final RtpEndUserState state) {
546 setWith(getWith(), state);
547 }
548
549 private void setWith(final Contact contact, final RtpEndUserState state) {
550 binding.with.setText(contact.getDisplayName());
551 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
552 .contains(state)) {
553 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
554 binding.withJid.setVisibility(View.VISIBLE);
555 } else {
556 binding.withJid.setVisibility(View.GONE);
557 }
558 }
559
560 private void proposeJingleRtpSession(
561 final Account account, final Jid with, final Set<Media> media) {
562 if (with.isBareJid()) {
563 xmppConnectionService
564 .getJingleConnectionManager()
565 .proposeJingleRtpSession(account, with, media);
566 } else {
567 throw new IllegalStateException(
568 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
569 }
570 putScreenInCallMode(media);
571 }
572
573 @Override
574 public void onRequestPermissionsResult(
575 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
576 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
577 final PermissionUtils.PermissionResult permissionResult =
578 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
579 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
580 if (requestCode == REQUEST_ACCEPT_CALL) {
581 acceptCall();
582 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
583 acceptContentAdd();
584 } else if (requestCode == REQUEST_ADD_CONTENT) {
585 switchToVideo();
586 }
587 } else {
588 @StringRes int res;
589 final String firstDenied =
590 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
591 if (firstDenied == null) {
592 return;
593 }
594 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
595 res = R.string.no_microphone_permission;
596 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
597 res = R.string.no_camera_permission;
598 } else {
599 throw new IllegalStateException("Invalid permission result request");
600 }
601 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
602 .show();
603 }
604 }
605
606 @Override
607 public void onStart() {
608 super.onStart();
609 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
610 this.binding.remoteVideo.setOnAspectRatioChanged(this);
611 }
612
613 @Override
614 public void onStop() {
615 mHandler.removeCallbacks(mTickExecutor);
616 binding.remoteVideo.release();
617 binding.remoteVideo.setOnAspectRatioChanged(null);
618 binding.localVideo.release();
619 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
620 final JingleRtpConnection jingleRtpConnection =
621 weakReference == null ? null : weakReference.get();
622 if (jingleRtpConnection != null) {
623 releaseVideoTracks(jingleRtpConnection);
624 }
625 releaseProximityWakeLock();
626 super.onStop();
627 }
628
629 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
630 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
631 if (remoteVideo.isPresent()) {
632 remoteVideo.get().removeSink(binding.remoteVideo);
633 }
634 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
635 if (localVideo.isPresent()) {
636 localVideo.get().removeSink(binding.localVideo);
637 }
638 }
639
640 @Override
641 public void onBackPressed() {
642 if (isConnected()) {
643 if (switchToPictureInPicture()) {
644 return;
645 }
646 } else {
647 endCall();
648 }
649 super.onBackPressed();
650 }
651
652 @Override
653 public void onUserLeaveHint() {
654 super.onUserLeaveHint();
655 if (switchToPictureInPicture()) {
656 return;
657 }
658 // TODO apparently this method is not getting called on Android 10 when using the task
659 // switcher
660 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
661 retractSessionProposal();
662 }
663 }
664
665 private boolean isConnected() {
666 final JingleRtpConnection connection =
667 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
668 final RtpEndUserState endUserState =
669 connection == null ? null : connection.getEndUserState();
670 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
671 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
672 }
673
674 private boolean switchToPictureInPicture() {
675 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
676 if (shouldBePictureInPicture()) {
677 startPictureInPicture();
678 return true;
679 }
680 }
681 return false;
682 }
683
684 @RequiresApi(api = Build.VERSION_CODES.O)
685 private void startPictureInPicture() {
686 try {
687 final Rational rational = this.binding.remoteVideo.getAspectRatio();
688 final Rational clippedRational = Rationals.clip(rational);
689 Log.d(
690 Config.LOGTAG,
691 "suggested rational " + rational + ". clipped to " + clippedRational);
692 enterPictureInPictureMode(
693 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
694 } catch (final IllegalStateException e) {
695 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
696 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
697 }
698 }
699
700 @Override
701 public void onAspectRatioChanged(final Rational rational) {
702 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
703 final Rational clippedRational = Rationals.clip(rational);
704 Log.d(
705 Config.LOGTAG,
706 "suggested rational after aspect ratio change "
707 + rational
708 + ". clipped to "
709 + clippedRational);
710 setPictureInPictureParams(
711 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
712 }
713 }
714
715 private boolean deviceSupportsPictureInPicture() {
716 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
717 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
718 } else {
719 return false;
720 }
721 }
722
723 private boolean shouldBePictureInPicture() {
724 try {
725 final JingleRtpConnection rtpConnection = requireRtpConnection();
726 return rtpConnection.getMedia().contains(Media.VIDEO)
727 && Arrays.asList(
728 RtpEndUserState.ACCEPTING_CALL,
729 RtpEndUserState.CONNECTING,
730 RtpEndUserState.CONNECTED)
731 .contains(rtpConnection.getEndUserState());
732 } catch (final IllegalStateException e) {
733 return false;
734 }
735 }
736
737 private boolean initializeActivityWithRunningRtpSession(
738 final Account account, Jid with, String sessionId) {
739 final WeakReference<JingleRtpConnection> reference =
740 xmppConnectionService
741 .getJingleConnectionManager()
742 .findJingleRtpConnection(account, with, sessionId);
743 if (reference == null || reference.get() == null) {
744 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
745 xmppConnectionService
746 .getJingleConnectionManager()
747 .getTerminalSessionState(with, sessionId);
748 if (terminatedRtpSession == null) {
749 throw new IllegalStateException(
750 "failed to initialize activity with running rtp session. session not found");
751 }
752 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
753 return true;
754 }
755 this.rtpConnectionReference = reference;
756 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
757 final boolean verified = requireRtpConnection().isVerified();
758 if (currentState == RtpEndUserState.ENDED) {
759 finish();
760 return true;
761 }
762 final Set<Media> media = getMedia();
763 final ContentAddition contentAddition = getPendingContentAddition();
764 if (currentState == RtpEndUserState.INCOMING_CALL) {
765 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
766 }
767 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
768 requireRtpConnection().getState())) {
769 putScreenInCallMode();
770 }
771 setWidth(currentState);
772 updateVideoViews(currentState);
773 updateStateDisplay(currentState, media, contentAddition);
774 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
775 updateButtonConfiguration(currentState, media, contentAddition);
776 updateIncomingCallScreen(currentState);
777 invalidateOptionsMenu();
778 return false;
779 }
780
781 private void initializeWithTerminatedSessionState(
782 final Account account,
783 final Jid with,
784 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
785 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
786 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
787 finish();
788 return;
789 }
790 final RtpEndUserState state = terminatedRtpSession.state;
791 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
792 updateButtonConfiguration(state);
793 updateStateDisplay(state);
794 updateIncomingCallScreen(state);
795 updateCallDuration();
796 updateVerifiedShield(false);
797 invalidateOptionsMenu();
798 setWith(account.getRoster().getContact(with), state);
799 }
800
801 private void reInitializeActivityWithRunningRtpSession(
802 final Account account, Jid with, String sessionId) {
803 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
804 resetIntent(account, with, sessionId);
805 }
806
807 private void resetIntent(final Account account, final Jid with, final String sessionId) {
808 final Intent intent = new Intent(Intent.ACTION_VIEW);
809 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
810 intent.putExtra(EXTRA_WITH, with.toEscapedString());
811 intent.putExtra(EXTRA_SESSION_ID, sessionId);
812 setIntent(intent);
813 }
814
815 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
816 surfaceViewRenderer.setVisibility(View.VISIBLE);
817 try {
818 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
819 } catch (final IllegalStateException ignored) {
820 // SurfaceViewRenderer was already initialized
821 } catch (final RuntimeException e) {
822 if (Throwables.getRootCause(e) instanceof GLException glException) {
823 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
824 }
825 }
826 surfaceViewRenderer.setEnableHardwareScaler(true);
827 }
828
829 private void updateStateDisplay(final RtpEndUserState state) {
830 updateStateDisplay(state, Collections.emptySet(), null);
831 }
832
833 private void updateStateDisplay(
834 final RtpEndUserState state,
835 final Set<Media> media,
836 final ContentAddition contentAddition) {
837 switch (state) {
838 case INCOMING_CALL -> {
839 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
840 if (media.contains(Media.VIDEO)) {
841 setTitle(R.string.rtp_state_incoming_video_call);
842 } else {
843 setTitle(R.string.rtp_state_incoming_call);
844 }
845 }
846 case INCOMING_CONTENT_ADD -> {
847 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
848 setTitle(R.string.rtp_state_content_add_video);
849 } else {
850 setTitle(R.string.rtp_state_content_add);
851 }
852 }
853 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
854 case CONNECTED -> setTitle(R.string.rtp_state_connected);
855 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
856 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
857 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
858 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
859 case RINGING -> setTitle(R.string.rtp_state_ringing);
860 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
861 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
862 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
863 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
864 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
865 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
866 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
867 case ENDED -> throw new IllegalStateException(
868 "Activity should have called finishAndReleaseWakeLock();");
869 default -> throw new IllegalStateException(
870 String.format("State %s has not been handled in UI", state));
871 }
872 }
873
874 private void updateVerifiedShield(final boolean verified) {
875 if (isPictureInPicture()) {
876 this.binding.verified.setVisibility(View.GONE);
877 return;
878 }
879 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
880 }
881
882 private void updateIncomingCallScreen(final RtpEndUserState state) {
883 updateIncomingCallScreen(state, null);
884 }
885
886 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
887 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
888 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
889 if (show) {
890 binding.contactPhoto.setVisibility(View.VISIBLE);
891 if (contact == null) {
892 AvatarWorkerTask.loadAvatar(
893 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
894 } else {
895 AvatarWorkerTask.loadAvatar(
896 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
897 }
898 } else {
899 binding.contactPhoto.setVisibility(View.GONE);
900 }
901 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
902 binding.usingAccount.setVisibility(View.VISIBLE);
903 binding.usingAccount.setText(
904 getString(
905 R.string.using_account,
906 account.getJid().asBareJid().toEscapedString()));
907 } else {
908 binding.usingAccount.setVisibility(View.GONE);
909 binding.contactPhoto.setVisibility(View.GONE);
910 }
911 }
912
913 private Set<Media> getMedia() {
914 return requireRtpConnection().getMedia();
915 }
916
917 public ContentAddition getPendingContentAddition() {
918 return requireRtpConnection().getPendingContentAddition();
919 }
920
921 private void updateButtonConfiguration(final RtpEndUserState state) {
922 updateButtonConfiguration(state, Collections.emptySet(), null);
923 }
924
925 @SuppressLint("RestrictedApi")
926 private void updateButtonConfiguration(
927 final RtpEndUserState state,
928 final Set<Media> media,
929 final ContentAddition contentAddition) {
930 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
931 this.binding.rejectCall.setVisibility(View.INVISIBLE);
932 this.binding.endCall.setVisibility(View.INVISIBLE);
933 this.binding.acceptCall.setVisibility(View.INVISIBLE);
934 } else if (state == RtpEndUserState.INCOMING_CALL) {
935 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
936 this.binding.rejectCall.setOnClickListener(this::rejectCall);
937 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
938 this.binding.rejectCall.setVisibility(View.VISIBLE);
939 this.binding.endCall.setVisibility(View.INVISIBLE);
940 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
941 this.binding.acceptCall.setOnClickListener(this::acceptCall);
942 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
943 this.binding.acceptCall.setVisibility(View.VISIBLE);
944 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
945 this.binding.rejectCall.setContentDescription(
946 getString(R.string.reject_switch_to_video));
947 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
948 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
949 this.binding.rejectCall.setVisibility(View.VISIBLE);
950 this.binding.endCall.setVisibility(View.INVISIBLE);
951 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
952 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
953 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
954 this.binding.acceptCall.setVisibility(View.VISIBLE);
955 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
956 .contains(state)) {
957 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
958 this.binding.rejectCall.setOnClickListener(this::exit);
959 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
960 this.binding.rejectCall.setVisibility(View.VISIBLE);
961 this.binding.endCall.setVisibility(View.INVISIBLE);
962 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
963 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
964 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
965 this.binding.acceptCall.setVisibility(View.VISIBLE);
966 } else if (asList(
967 RtpEndUserState.CONNECTIVITY_ERROR,
968 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
969 RtpEndUserState.APPLICATION_ERROR,
970 RtpEndUserState.RETRACTED,
971 RtpEndUserState.SECURITY_ERROR)
972 .contains(state)) {
973 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
974 this.binding.rejectCall.setOnClickListener(this::exit);
975 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
976 this.binding.rejectCall.setVisibility(View.VISIBLE);
977 this.binding.endCall.setVisibility(View.INVISIBLE);
978 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
979 this.binding.acceptCall.setOnClickListener(this::retry);
980 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
981 this.binding.acceptCall.setVisibility(View.VISIBLE);
982 } else {
983 this.binding.rejectCall.setVisibility(View.INVISIBLE);
984 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
985 this.binding.endCall.setOnClickListener(this::endCall);
986 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
987 this.binding.endCall.setVisibility(View.VISIBLE);
988 this.binding.acceptCall.setVisibility(View.INVISIBLE);
989 }
990 updateInCallButtonConfiguration(state, media);
991 }
992
993 private boolean isPictureInPicture() {
994 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
995 return isInPictureInPictureMode();
996 } else {
997 return false;
998 }
999 }
1000
1001 private void updateInCallButtonConfiguration() {
1002 updateInCallButtonConfiguration(
1003 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1004 }
1005
1006 @SuppressLint("RestrictedApi")
1007 private void updateInCallButtonConfiguration(
1008 final RtpEndUserState state, final Set<Media> media) {
1009 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1010 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1011 if (media.contains(Media.VIDEO)) {
1012 final JingleRtpConnection rtpConnection = requireRtpConnection();
1013 updateInCallButtonConfigurationVideo(
1014 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1015 } else {
1016 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1017 updateInCallButtonConfigurationSpeaker(
1018 callIntegration.getSelectedAudioDevice(),
1019 callIntegration.getAudioDevices().size());
1020 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1021 }
1022 if (media.contains(Media.AUDIO)) {
1023 updateInCallButtonConfigurationMicrophone(
1024 requireRtpConnection().isMicrophoneEnabled());
1025 } else {
1026 this.binding.inCallActionLeft.setVisibility(View.GONE);
1027 }
1028 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1029 && !isPictureInPicture()
1030 && Media.audioOnly(media)) {
1031 final CallIntegration callIntegration = requireCallIntegration();
1032 updateInCallButtonConfigurationSpeaker(
1033 callIntegration.getSelectedAudioDevice(),
1034 callIntegration.getAudioDevices().size());
1035 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1036 } else {
1037 this.binding.inCallActionLeft.setVisibility(View.GONE);
1038 this.binding.inCallActionRight.setVisibility(View.GONE);
1039 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1040 }
1041 }
1042
1043 @SuppressLint("RestrictedApi")
1044 private void updateInCallButtonConfigurationSpeaker(
1045 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1046 switch (selectedAudioDevice) {
1047 case EARPIECE -> {
1048 this.binding.inCallActionRight.setImageResource(
1049 R.drawable.ic_volume_off_black_24dp);
1050 if (numberOfChoices >= 2) {
1051 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1052 } else {
1053 this.binding.inCallActionRight.setOnClickListener(null);
1054 this.binding.inCallActionRight.setClickable(false);
1055 }
1056 }
1057 case WIRED_HEADSET -> {
1058 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1059 this.binding.inCallActionRight.setOnClickListener(null);
1060 this.binding.inCallActionRight.setClickable(false);
1061 }
1062 case SPEAKER_PHONE -> {
1063 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1064 if (numberOfChoices >= 2) {
1065 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1066 } else {
1067 this.binding.inCallActionRight.setOnClickListener(null);
1068 this.binding.inCallActionRight.setClickable(false);
1069 }
1070 }
1071 case BLUETOOTH -> {
1072 this.binding.inCallActionRight.setImageResource(
1073 R.drawable.ic_bluetooth_audio_black_24dp);
1074 this.binding.inCallActionRight.setOnClickListener(null);
1075 this.binding.inCallActionRight.setClickable(false);
1076 }
1077 }
1078 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1079 }
1080
1081 @SuppressLint("RestrictedApi")
1082 private void updateInCallButtonConfigurationVideo(
1083 final boolean videoEnabled, final boolean isCameraSwitchable) {
1084 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1085 if (isCameraSwitchable) {
1086 this.binding.inCallActionFarRight.setImageResource(
1087 R.drawable.ic_flip_camera_android_black_24dp);
1088 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1089 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1090 } else {
1091 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1092 }
1093 if (videoEnabled) {
1094 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1095 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1096 } else {
1097 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1098 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1099 }
1100 }
1101
1102 private void switchCamera(final View view) {
1103 Futures.addCallback(
1104 requireRtpConnection().switchCamera(),
1105 new FutureCallback<>() {
1106 @Override
1107 public void onSuccess(@Nullable Boolean isFrontCamera) {
1108 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1109 }
1110
1111 @Override
1112 public void onFailure(@NonNull final Throwable throwable) {
1113 Log.d(
1114 Config.LOGTAG,
1115 "could not switch camera",
1116 Throwables.getRootCause(throwable));
1117 Toast.makeText(
1118 RtpSessionActivity.this,
1119 R.string.could_not_switch_camera,
1120 Toast.LENGTH_LONG)
1121 .show();
1122 }
1123 },
1124 MainThreadExecutor.getInstance());
1125 }
1126
1127 private void enableVideo(View view) {
1128 try {
1129 requireRtpConnection().setVideoEnabled(true);
1130 } catch (final IllegalStateException e) {
1131 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1132 return;
1133 }
1134 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1135 }
1136
1137 private void disableVideo(View view) {
1138 final JingleRtpConnection rtpConnection = requireRtpConnection();
1139 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1140 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1141 rtpConnection.retractContentAdd();
1142 return;
1143 }
1144 requireRtpConnection().setVideoEnabled(false);
1145 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1146 }
1147
1148 @SuppressLint("RestrictedApi")
1149 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1150 if (microphoneEnabled) {
1151 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1152 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1153 } else {
1154 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1155 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1156 }
1157 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1158 }
1159
1160 private void updateCallDuration() {
1161 final JingleRtpConnection connection =
1162 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1163 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1164 this.binding.duration.setVisibility(View.GONE);
1165 return;
1166 }
1167 if (connection.zeroDuration()) {
1168 this.binding.duration.setVisibility(View.GONE);
1169 } else {
1170 this.binding.duration.setText(
1171 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1172 this.binding.duration.setVisibility(View.VISIBLE);
1173 }
1174 }
1175
1176 private void updateVideoViews(final RtpEndUserState state) {
1177 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1178 binding.localVideo.setVisibility(View.GONE);
1179 binding.localVideo.release();
1180 binding.remoteVideoWrapper.setVisibility(View.GONE);
1181 binding.remoteVideo.release();
1182 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1183 if (isPictureInPicture()) {
1184 binding.appBarLayout.setVisibility(View.GONE);
1185 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1186 if (Arrays.asList(
1187 RtpEndUserState.APPLICATION_ERROR,
1188 RtpEndUserState.CONNECTIVITY_ERROR,
1189 RtpEndUserState.SECURITY_ERROR)
1190 .contains(state)) {
1191 binding.pipWarning.setVisibility(View.VISIBLE);
1192 binding.pipWaiting.setVisibility(View.GONE);
1193 } else {
1194 binding.pipWarning.setVisibility(View.GONE);
1195 binding.pipWaiting.setVisibility(View.GONE);
1196 }
1197 } else {
1198 binding.appBarLayout.setVisibility(View.VISIBLE);
1199 binding.pipPlaceholder.setVisibility(View.GONE);
1200 }
1201 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1202 return;
1203 }
1204 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1205 binding.localVideo.setVisibility(View.GONE);
1206 binding.remoteVideoWrapper.setVisibility(View.GONE);
1207 binding.appBarLayout.setVisibility(View.GONE);
1208 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1209 binding.pipWarning.setVisibility(View.GONE);
1210 binding.pipWaiting.setVisibility(View.VISIBLE);
1211 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1212 return;
1213 }
1214 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1215 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1216 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1217 // paint local view over remote view
1218 binding.localVideo.setZOrderMediaOverlay(true);
1219 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1220 addSink(localVideoTrack.get(), binding.localVideo);
1221 } else {
1222 binding.localVideo.setVisibility(View.GONE);
1223 }
1224 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1225 if (remoteVideoTrack.isPresent()) {
1226 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1227 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1228 binding.remoteVideo.setScalingType(
1229 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1230 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1231 if (state == RtpEndUserState.CONNECTED) {
1232 binding.appBarLayout.setVisibility(View.GONE);
1233 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1234 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1235 } else {
1236 binding.appBarLayout.setVisibility(View.VISIBLE);
1237 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1238 binding.remoteVideoWrapper.setVisibility(View.GONE);
1239 }
1240 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1241 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1242 } else {
1243 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1244 }
1245 } else {
1246 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1247 binding.remoteVideoWrapper.setVisibility(View.GONE);
1248 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1249 }
1250 }
1251
1252 private Optional<VideoTrack> getLocalVideoTrack() {
1253 final JingleRtpConnection connection =
1254 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1255 if (connection == null) {
1256 return Optional.absent();
1257 }
1258 return connection.getLocalVideoTrack();
1259 }
1260
1261 private Optional<VideoTrack> getRemoteVideoTrack() {
1262 final JingleRtpConnection connection =
1263 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1264 if (connection == null) {
1265 return Optional.absent();
1266 }
1267 return connection.getRemoteVideoTrack();
1268 }
1269
1270 private void disableMicrophone(View view) {
1271 final JingleRtpConnection rtpConnection = requireRtpConnection();
1272 if (rtpConnection.setMicrophoneEnabled(false)) {
1273 updateInCallButtonConfiguration();
1274 }
1275 }
1276
1277 private void enableMicrophone(View view) {
1278 final JingleRtpConnection rtpConnection = requireRtpConnection();
1279 if (rtpConnection.setMicrophoneEnabled(true)) {
1280 updateInCallButtonConfiguration();
1281 }
1282 }
1283
1284 private void switchToEarpiece(final View view) {
1285 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1286 acquireProximityWakeLock();
1287 }
1288
1289 private void switchToSpeaker(final View view) {
1290 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1291 releaseProximityWakeLock();
1292 }
1293
1294 private void retry(final View view) {
1295 final Intent intent = getIntent();
1296 final Account account = extractAccount(intent);
1297 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1298 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1299 final String action = intent.getAction();
1300 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1301 this.rtpConnectionReference = null;
1302 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1303 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1304 }
1305
1306 private void exit(final View view) {
1307 finish();
1308 }
1309
1310 private void recordVoiceMail(final View view) {
1311 final Intent intent = getIntent();
1312 final Account account = extractAccount(intent);
1313 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1314 final Conversation conversation =
1315 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1316 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1317 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1318 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1319 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1320 launchIntent.putExtra(
1321 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1322 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1323 startActivity(launchIntent);
1324 finish();
1325 }
1326
1327 private Contact getWith() {
1328 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1329 final Account account = id.account;
1330 return account.getRoster().getContact(id.with);
1331 }
1332
1333 private JingleRtpConnection requireRtpConnection() {
1334 final JingleRtpConnection connection =
1335 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1336 if (connection == null) {
1337 throw new IllegalStateException("No RTP connection found");
1338 }
1339 return connection;
1340 }
1341
1342 private CallIntegration requireCallIntegration() {
1343 return requireOngoingRtpSession().getCallIntegration();
1344 }
1345
1346 private OngoingRtpSession requireOngoingRtpSession() {
1347 final JingleRtpConnection connection =
1348 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1349 if (connection != null) {
1350 return connection;
1351 }
1352 final Intent currentIntent = getIntent();
1353 final String withExtra =
1354 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1355 final var account = extractAccount(currentIntent);
1356 if (withExtra == null) {
1357 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1358 }
1359 final var matching =
1360 xmppConnectionService
1361 .getJingleConnectionManager()
1362 .matchingProposal(account, Jid.of(withExtra));
1363 if (matching.isPresent()) {
1364 return matching.get();
1365 }
1366 throw new IllegalStateException("No matching session proposal");
1367 }
1368
1369 @Override
1370 public void onJingleRtpConnectionUpdate(
1371 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1372 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1373 if (END_CARD.contains(state)) {
1374 Log.d(Config.LOGTAG, "end card reached");
1375 releaseProximityWakeLock();
1376 runOnUiThread(
1377 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1378 }
1379 if (with.isBareJid()) {
1380 updateRtpSessionProposalState(account, with, state);
1381 return;
1382 }
1383 if (emptyReference(this.rtpConnectionReference)) {
1384 if (END_CARD.contains(state)) {
1385 Log.d(Config.LOGTAG, "not reinitializing session");
1386 return;
1387 }
1388 // this happens when going from proposed session to actual session
1389 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1390 return;
1391 }
1392 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1393 final boolean verified = requireRtpConnection().isVerified();
1394 final Set<Media> media = getMedia();
1395 lockOrientation(media);
1396 final ContentAddition contentAddition = getPendingContentAddition();
1397 final Contact contact = getWith();
1398 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1399 if (state == RtpEndUserState.ENDED) {
1400 finish();
1401 return;
1402 }
1403 runOnUiThread(
1404 () -> {
1405 updateStateDisplay(state, media, contentAddition);
1406 updateVerifiedShield(
1407 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1408 updateButtonConfiguration(state, media, contentAddition);
1409 updateVideoViews(state);
1410 updateIncomingCallScreen(state, contact);
1411 invalidateOptionsMenu();
1412 });
1413 if (END_CARD.contains(state)) {
1414 final JingleRtpConnection rtpConnection = requireRtpConnection();
1415 resetIntent(account, with, state, rtpConnection.getMedia());
1416 releaseVideoTracks(rtpConnection);
1417 this.rtpConnectionReference = null;
1418 }
1419 } else {
1420 Log.d(Config.LOGTAG, "received update for other rtp session");
1421 }
1422 }
1423
1424 @Override
1425 public void onAudioDeviceChanged(
1426 final CallIntegration.AudioDevice selectedAudioDevice,
1427 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1428 Log.d(
1429 Config.LOGTAG,
1430 "onAudioDeviceChanged in activity: selected:"
1431 + selectedAudioDevice
1432 + ", available:"
1433 + availableAudioDevices);
1434 try {
1435 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1436 final RtpEndUserState endUserState;
1437 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1438 endUserState = jingleRtpConnection.getEndUserState();
1439 } else {
1440 // for session proposals all end user states are functionally the same
1441 endUserState = RtpEndUserState.RINGING;
1442 }
1443 final Set<Media> media = ongoingRtpSession.getMedia();
1444 if (END_CARD.contains(endUserState)) {
1445 Log.d(
1446 Config.LOGTAG,
1447 "onAudioDeviceChanged() nothing to do because end card has been reached");
1448 } else {
1449 if (Media.audioOnly(media)
1450 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1451 final CallIntegration callIntegration = requireCallIntegration();
1452 updateInCallButtonConfigurationSpeaker(
1453 callIntegration.getSelectedAudioDevice(),
1454 callIntegration.getAudioDevices().size());
1455 }
1456 Log.d(
1457 Config.LOGTAG,
1458 "put proximity wake lock into proper state after device update");
1459 putProximityWakeLockInProperState(selectedAudioDevice);
1460 }
1461 } catch (final IllegalStateException e) {
1462 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1463 }
1464 }
1465
1466 private void updateRtpSessionProposalState(
1467 final Account account, final Jid with, final RtpEndUserState state) {
1468 final Intent currentIntent = getIntent();
1469 final String withExtra =
1470 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1471 if (withExtra == null) {
1472 return;
1473 }
1474 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1475 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1476 runOnUiThread(
1477 () -> {
1478 updateVerifiedShield(false);
1479 updateStateDisplay(state);
1480 updateButtonConfiguration(state, media, null);
1481 updateIncomingCallScreen(state);
1482 invalidateOptionsMenu();
1483 });
1484 resetIntent(account, with, state, media);
1485 }
1486 }
1487
1488 private void resetIntent(final Bundle extras) {
1489 final Intent intent = new Intent(Intent.ACTION_VIEW);
1490 intent.putExtras(extras);
1491 setIntent(intent);
1492 }
1493
1494 private void resetIntent(
1495 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1496 final Intent intent = new Intent(Intent.ACTION_VIEW);
1497 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1498 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1499 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1500 } else {
1501 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1502 }
1503 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1504 intent.putExtra(
1505 EXTRA_LAST_ACTION,
1506 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1507 setIntent(intent);
1508 }
1509
1510 private static boolean emptyReference(final WeakReference<?> weakReference) {
1511 return weakReference == null || weakReference.get() == null;
1512 }
1513}