1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.opengl.GLException;
17import android.os.Build;
18import android.os.Bundle;
19import android.os.Handler;
20import android.os.PowerManager;
21import android.os.SystemClock;
22import android.util.Log;
23import android.util.Rational;
24import android.view.KeyEvent;
25import android.view.Menu;
26import android.view.MenuItem;
27import android.view.View;
28import android.view.WindowManager;
29import android.widget.Toast;
30
31import androidx.annotation.NonNull;
32import androidx.annotation.Nullable;
33import androidx.annotation.RequiresApi;
34import androidx.annotation.StringRes;
35import androidx.databinding.DataBindingUtil;
36
37import com.google.common.base.Optional;
38import com.google.common.base.Preconditions;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.services.CallIntegration;
53import eu.siacs.conversations.services.CallIntegrationConnectionService;
54import eu.siacs.conversations.services.XmppConnectionService;
55import eu.siacs.conversations.ui.util.AvatarWorkerTask;
56import eu.siacs.conversations.ui.util.MainThreadExecutor;
57import eu.siacs.conversations.ui.util.Rationals;
58import eu.siacs.conversations.utils.PermissionUtils;
59import eu.siacs.conversations.utils.TimeFrameUtils;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
62import eu.siacs.conversations.xmpp.jingle.ContentAddition;
63import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
64import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
65import eu.siacs.conversations.xmpp.jingle.Media;
66import eu.siacs.conversations.xmpp.jingle.RtpCapability;
67import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
68
69import org.webrtc.RendererCommon;
70import org.webrtc.SurfaceViewRenderer;
71import org.webrtc.VideoTrack;
72
73import java.lang.ref.WeakReference;
74import java.util.Arrays;
75import java.util.Collections;
76import java.util.List;
77import java.util.Set;
78
79public class RtpSessionActivity extends XmppActivity
80 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
81 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
82
83 public static final String EXTRA_WITH = "with";
84 public static final String EXTRA_SESSION_ID = "session_id";
85 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
86 public static final String EXTRA_LAST_ACTION = "last_action";
87 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
88 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
89 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
90
91 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
92
93 private static final List<RtpEndUserState> END_CARD =
94 Arrays.asList(
95 RtpEndUserState.APPLICATION_ERROR,
96 RtpEndUserState.SECURITY_ERROR,
97 RtpEndUserState.DECLINED_OR_BUSY,
98 RtpEndUserState.CONNECTIVITY_ERROR,
99 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
100 RtpEndUserState.RETRACTED);
101 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
102 Arrays.asList(
103 RtpEndUserState.APPLICATION_ERROR,
104 RtpEndUserState.CONNECTIVITY_ERROR,
105 RtpEndUserState.SECURITY_ERROR);
106 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
107 Arrays.asList(
108 RtpEndUserState.CONNECTING,
109 RtpEndUserState.CONNECTED,
110 RtpEndUserState.RECONNECTING,
111 RtpEndUserState.INCOMING_CONTENT_ADD);
112 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
113 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
114 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
115 Arrays.asList(
116 RtpEndUserState.ACCEPTING_CALL,
117 RtpEndUserState.CONNECTING,
118 RtpEndUserState.RECONNECTING);
119 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
120 private static final int REQUEST_ACCEPT_CALL = 0x1111;
121 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
122 private static final int REQUEST_ADD_CONTENT = 0x1113;
123 private WeakReference<JingleRtpConnection> rtpConnectionReference;
124
125 private ActivityRtpSessionBinding binding;
126 private PowerManager.WakeLock mProximityWakeLock;
127
128 private final Handler mHandler = new Handler();
129 private final Runnable mTickExecutor =
130 new Runnable() {
131 @Override
132 public void run() {
133 updateCallDuration();
134 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
135 }
136 };
137
138 public static Set<Media> actionToMedia(final String action) {
139 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
140 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
141 } else {
142 return ImmutableSet.of(Media.AUDIO);
143 }
144 }
145
146 private static void addSink(
147 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
148 try {
149 videoTrack.addSink(surfaceViewRenderer);
150 } catch (final IllegalStateException e) {
151 Log.e(
152 Config.LOGTAG,
153 "possible race condition on trying to display video track. ignoring",
154 e);
155 }
156 }
157
158 @Override
159 public void onCreate(Bundle savedInstanceState) {
160 super.onCreate(savedInstanceState);
161 getWindow()
162 .addFlags(
163 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
164 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
165 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
166 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
167 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
168 setSupportActionBar(binding.toolbar);
169 }
170
171 @Override
172 public boolean onCreateOptionsMenu(final Menu menu) {
173 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
174 final MenuItem help = menu.findItem(R.id.action_help);
175 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
176 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
177 help.setVisible(Config.HELP != null && isHelpButtonVisible());
178 gotoChat.setVisible(isSwitchToConversationVisible());
179 switchToVideo.setVisible(isSwitchToVideoVisible());
180 return super.onCreateOptionsMenu(menu);
181 }
182
183 @Override
184 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
185 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
186 if (xmppConnectionService != null) {
187 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
188 return true;
189 }
190 }
191 }
192 return super.onKeyDown(keyCode, event);
193 }
194
195 private boolean isHelpButtonVisible() {
196 try {
197 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
198 } catch (IllegalStateException e) {
199 final Intent intent = getIntent();
200 final String state =
201 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
202 if (state != null) {
203 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
204 } else {
205 return false;
206 }
207 }
208 }
209
210 private boolean isSwitchToConversationVisible() {
211 final JingleRtpConnection connection =
212 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
213 return connection != null
214 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
215 }
216
217 private boolean isSwitchToVideoVisible() {
218 final JingleRtpConnection connection =
219 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
220 if (connection == null) {
221 return false;
222 }
223 return connection.isSwitchToVideoAvailable();
224 }
225
226 private void switchToConversation() {
227 final Contact contact = getWith();
228 final Conversation conversation =
229 xmppConnectionService.findOrCreateConversation(
230 contact.getAccount(), contact.getJid(), false, true);
231 switchToConversation(conversation);
232 }
233
234 public boolean onOptionsItemSelected(final MenuItem item) {
235 final var itemItem = item.getItemId();
236 if (itemItem == R.id.action_help) {
237 launchHelpInBrowser();
238 return true;
239 } else if (itemItem == R.id.action_goto_chat) {
240 switchToConversation();
241 return true;
242 } else if (itemItem == R.id.action_switch_to_video) {
243 requestPermissionAndSwitchToVideo();
244 return true;
245 } else {
246 return super.onOptionsItemSelected(item);
247 }
248 }
249
250 private void launchHelpInBrowser() {
251 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
252 try {
253 startActivity(intent);
254 } catch (final ActivityNotFoundException e) {
255 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
256 .show();
257 }
258 }
259
260 private void endCall(View view) {
261 endCall();
262 }
263
264 private void endCall() {
265 if (this.rtpConnectionReference == null) {
266 retractSessionProposal();
267 finish();
268 } else {
269 requireRtpConnection().endCall();
270 }
271 }
272
273 private void retractSessionProposal() {
274 final Intent intent = getIntent();
275 final String action = intent.getAction();
276 final Account account = extractAccount(intent);
277 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
278 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
279 if (!Intent.ACTION_VIEW.equals(action)
280 || state == null
281 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
282 resetIntent(
283 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
284 }
285 xmppConnectionService
286 .getJingleConnectionManager()
287 .retractSessionProposal(account, with.asBareJid());
288 }
289
290 private void rejectCall(View view) {
291 requireRtpConnection().rejectCall();
292 finish();
293 }
294
295 private void acceptCall(View view) {
296 requestPermissionsAndAcceptCall();
297 }
298
299 private void acceptContentAdd() {
300 try {
301 requireRtpConnection()
302 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
303 } catch (final IllegalStateException e) {
304 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
305 }
306 }
307
308 private void requestPermissionAndSwitchToVideo() {
309 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
310 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
311 switchToVideo();
312 }
313 }
314
315 private void switchToVideo() {
316 try {
317 requireRtpConnection().addMedia(Media.VIDEO);
318 } catch (final IllegalStateException e) {
319 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
320 }
321 }
322
323 private void acceptContentAdd(final ContentAddition contentAddition) {
324 if (contentAddition == null
325 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
326 Log.d(Config.LOGTAG, "ignore press on content-accept button");
327 return;
328 }
329 requestPermissionAndAcceptContentAdd(contentAddition);
330 }
331
332 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
333 final List<String> permissions = permissions(contentAddition.media());
334 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
335 try {
336 requireRtpConnection().acceptContentAdd(contentAddition.summary);
337 } catch (final IllegalStateException e) {
338 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
339 }
340 }
341 }
342
343 private void rejectContentAdd(final View view) {
344 requireRtpConnection().rejectContentAdd();
345 }
346
347 private void requestPermissionsAndAcceptCall() {
348 final List<String> permissions = permissions(getMedia());
349 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
350 putScreenInCallMode();
351 checkRecorderAndAcceptCall();
352 }
353 }
354
355 private List<String> permissions(final Set<Media> media) {
356 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
357 if (media.contains(Media.VIDEO)) {
358 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
359 } else {
360 permissions.add(Manifest.permission.RECORD_AUDIO);
361 }
362 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
363 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
364 }
365 return permissions.build();
366 }
367
368 private void checkRecorderAndAcceptCall() {
369 checkMicrophoneAvailabilityAsync();
370 try {
371 requireRtpConnection().acceptCall();
372 } catch (final IllegalStateException e) {
373 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
374 }
375 }
376
377 private void checkMicrophoneAvailabilityAsync() {
378 new Thread(new MicrophoneAvailabilityCheck(this)).start();
379 }
380
381 private static class MicrophoneAvailabilityCheck implements Runnable {
382
383 private final WeakReference<Activity> activityReference;
384
385 private MicrophoneAvailabilityCheck(final Activity activity) {
386 this.activityReference = new WeakReference<>(activity);
387 }
388
389 @Override
390 public void run() {
391 final long start = SystemClock.elapsedRealtime();
392 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
393 final long stop = SystemClock.elapsedRealtime();
394 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
395 if (isMicrophoneAvailable) {
396 return;
397 }
398 final Activity activity = activityReference.get();
399 if (activity == null) {
400 return;
401 }
402 activity.runOnUiThread(
403 () ->
404 Toast.makeText(
405 activity,
406 R.string.microphone_unavailable,
407 Toast.LENGTH_LONG)
408 .show());
409 }
410 }
411
412 private void putScreenInCallMode() {
413 putScreenInCallMode(requireRtpConnection().getMedia());
414 }
415
416 private void putScreenInCallMode(final Set<Media> media) {
417 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
418 if (Media.audioOnly(media)) {
419 final JingleRtpConnection rtpConnection =
420 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
421 final CallIntegration callIntegration =
422 rtpConnection == null ? null : rtpConnection.getCallIntegration();
423 if (callIntegration == null
424 || callIntegration.getSelectedAudioDevice()
425 == CallIntegration.AudioDevice.EARPIECE) {
426 acquireProximityWakeLock();
427 }
428 }
429 lockOrientation(media);
430 }
431
432 private void lockOrientation(final Set<Media> media) {
433 if (Media.audioOnly(media)) {
434 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
435 } else {
436 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
437 }
438 }
439
440 @SuppressLint("WakelockTimeout")
441 private void acquireProximityWakeLock() {
442 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
443 if (powerManager == null) {
444 Log.e(Config.LOGTAG, "power manager not available");
445 return;
446 }
447 if (isFinishing()) {
448 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
449 return;
450 }
451 if (this.mProximityWakeLock == null) {
452 this.mProximityWakeLock =
453 powerManager.newWakeLock(
454 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
455 }
456 if (!this.mProximityWakeLock.isHeld()) {
457 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
458 this.mProximityWakeLock.acquire();
459 }
460 }
461
462 private void releaseProximityWakeLock() {
463 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
464 Log.d(Config.LOGTAG, "releasing proximity wake lock");
465 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
466 this.mProximityWakeLock = null;
467 }
468 }
469
470 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
471 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
472 acquireProximityWakeLock();
473 } else {
474 releaseProximityWakeLock();
475 }
476 }
477
478 @Override
479 protected void refreshUiReal() {}
480
481 @Override
482 public void onNewIntent(final Intent intent) {
483 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
484 super.onNewIntent(intent);
485 setIntent(intent);
486 if (xmppConnectionService == null) {
487 Log.d(
488 Config.LOGTAG,
489 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
490 return;
491 }
492 final Account account = extractAccount(intent);
493 final String action = intent.getAction();
494 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
495 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
496 if (sessionId != null) {
497 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
498 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
499 return;
500 }
501 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
502 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
503 requestPermissionsAndAcceptCall();
504 resetIntent(intent.getExtras());
505 }
506 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
507 proposeJingleRtpSession(account, with, actionToMedia(action));
508 setWith(account.getRoster().getContact(with), null);
509 } else {
510 throw new IllegalStateException("received onNewIntent without sessionId");
511 }
512 }
513
514 @Override
515 void onBackendConnected() {
516 final Intent intent = getIntent();
517 final String action = intent.getAction();
518 final Account account = extractAccount(intent);
519 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
520 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
521 if (sessionId != null) {
522 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
523 return;
524 }
525 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
526 Log.d(Config.LOGTAG, "intent action was accept");
527 requestPermissionsAndAcceptCall();
528 resetIntent(intent.getExtras());
529 }
530 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
531 proposeJingleRtpSession(account, with, actionToMedia(action));
532 setWith(account.getRoster().getContact(with), null);
533 } else if (Intent.ACTION_VIEW.equals(action)) {
534 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
535 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
536 final Set<Media> media = actionToMedia(lastAction);
537 final RtpEndUserState state =
538 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
539 if (state != null) {
540 Log.d(Config.LOGTAG, "restored last state from intent extra");
541 updateButtonConfiguration(state);
542 updateVerifiedShield(false);
543 updateStateDisplay(state);
544 updateIncomingCallScreen(state);
545 invalidateOptionsMenu();
546 }
547 setWith(account.getRoster().getContact(with), state);
548 if (xmppConnectionService
549 .getJingleConnectionManager()
550 .fireJingleRtpConnectionStateUpdates()) {
551 return;
552 }
553 if (END_CARD.contains(state)) {
554 return;
555 }
556 if (xmppConnectionService
557 .getJingleConnectionManager()
558 .hasMatchingProposal(account, with)) {
559 putScreenInCallMode(media);
560 return;
561 }
562 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
563 finish();
564 }
565 }
566
567 private void setWidth(final RtpEndUserState state) {
568 setWith(getWith(), state);
569 }
570
571 private void setWith(final Contact contact, final RtpEndUserState state) {
572 binding.with.setText(contact.getDisplayName());
573 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
574 .contains(state)) {
575 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
576 binding.withJid.setVisibility(View.VISIBLE);
577 } else {
578 binding.withJid.setVisibility(View.GONE);
579 }
580 }
581
582 private void proposeJingleRtpSession(
583 final Account account, final Jid with, final Set<Media> media) {
584 checkMicrophoneAvailabilityAsync();
585 if (with.isBareJid()) {
586 xmppConnectionService
587 .getJingleConnectionManager()
588 .proposeJingleRtpSession(account, with, media);
589 } else {
590 throw new IllegalStateException(
591 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
592 }
593 putScreenInCallMode(media);
594 }
595
596 @Override
597 public void onRequestPermissionsResult(
598 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
599 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
600 final PermissionUtils.PermissionResult permissionResult =
601 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
602 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
603 if (requestCode == REQUEST_ACCEPT_CALL) {
604 checkRecorderAndAcceptCall();
605 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
606 acceptContentAdd();
607 } else if (requestCode == REQUEST_ADD_CONTENT) {
608 switchToVideo();
609 }
610 } else {
611 @StringRes int res;
612 final String firstDenied =
613 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
614 if (firstDenied == null) {
615 return;
616 }
617 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
618 res = R.string.no_microphone_permission;
619 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
620 res = R.string.no_camera_permission;
621 } else {
622 throw new IllegalStateException("Invalid permission result request");
623 }
624 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
625 .show();
626 }
627 }
628
629 @Override
630 public void onStart() {
631 super.onStart();
632 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
633 this.binding.remoteVideo.setOnAspectRatioChanged(this);
634 }
635
636 @Override
637 public void onStop() {
638 mHandler.removeCallbacks(mTickExecutor);
639 binding.remoteVideo.release();
640 binding.remoteVideo.setOnAspectRatioChanged(null);
641 binding.localVideo.release();
642 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
643 final JingleRtpConnection jingleRtpConnection =
644 weakReference == null ? null : weakReference.get();
645 if (jingleRtpConnection != null) {
646 releaseVideoTracks(jingleRtpConnection);
647 }
648 releaseProximityWakeLock();
649 super.onStop();
650 }
651
652 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
653 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
654 if (remoteVideo.isPresent()) {
655 remoteVideo.get().removeSink(binding.remoteVideo);
656 }
657 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
658 if (localVideo.isPresent()) {
659 localVideo.get().removeSink(binding.localVideo);
660 }
661 }
662
663 @Override
664 public void onBackPressed() {
665 if (isConnected()) {
666 if (switchToPictureInPicture()) {
667 return;
668 }
669 } else {
670 endCall();
671 }
672 super.onBackPressed();
673 }
674
675 @Override
676 public void onUserLeaveHint() {
677 super.onUserLeaveHint();
678 if (switchToPictureInPicture()) {
679 return;
680 }
681 // TODO apparently this method is not getting called on Android 10 when using the task
682 // switcher
683 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
684 retractSessionProposal();
685 }
686 }
687
688 private boolean isConnected() {
689 final JingleRtpConnection connection =
690 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
691 final RtpEndUserState endUserState =
692 connection == null ? null : connection.getEndUserState();
693 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
694 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
695 }
696
697 private boolean switchToPictureInPicture() {
698 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
699 if (shouldBePictureInPicture()) {
700 startPictureInPicture();
701 return true;
702 }
703 }
704 return false;
705 }
706
707 @RequiresApi(api = Build.VERSION_CODES.O)
708 private void startPictureInPicture() {
709 try {
710 final Rational rational = this.binding.remoteVideo.getAspectRatio();
711 final Rational clippedRational = Rationals.clip(rational);
712 Log.d(
713 Config.LOGTAG,
714 "suggested rational " + rational + ". clipped to " + clippedRational);
715 enterPictureInPictureMode(
716 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
717 } catch (final IllegalStateException e) {
718 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
719 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
720 }
721 }
722
723 @Override
724 public void onAspectRatioChanged(final Rational rational) {
725 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
726 final Rational clippedRational = Rationals.clip(rational);
727 Log.d(
728 Config.LOGTAG,
729 "suggested rational after aspect ratio change "
730 + rational
731 + ". clipped to "
732 + clippedRational);
733 setPictureInPictureParams(
734 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
735 }
736 }
737
738 private boolean deviceSupportsPictureInPicture() {
739 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
740 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
741 } else {
742 return false;
743 }
744 }
745
746 private boolean shouldBePictureInPicture() {
747 try {
748 final JingleRtpConnection rtpConnection = requireRtpConnection();
749 return rtpConnection.getMedia().contains(Media.VIDEO)
750 && Arrays.asList(
751 RtpEndUserState.ACCEPTING_CALL,
752 RtpEndUserState.CONNECTING,
753 RtpEndUserState.CONNECTED)
754 .contains(rtpConnection.getEndUserState());
755 } catch (final IllegalStateException e) {
756 return false;
757 }
758 }
759
760 private boolean initializeActivityWithRunningRtpSession(
761 final Account account, Jid with, String sessionId) {
762 final WeakReference<JingleRtpConnection> reference =
763 xmppConnectionService
764 .getJingleConnectionManager()
765 .findJingleRtpConnection(account, with, sessionId);
766 if (reference == null || reference.get() == null) {
767 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
768 xmppConnectionService
769 .getJingleConnectionManager()
770 .getTerminalSessionState(with, sessionId);
771 if (terminatedRtpSession == null) {
772 throw new IllegalStateException(
773 "failed to initialize activity with running rtp session. session not found");
774 }
775 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
776 return true;
777 }
778 this.rtpConnectionReference = reference;
779 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
780 final boolean verified = requireRtpConnection().isVerified();
781 if (currentState == RtpEndUserState.ENDED) {
782 finish();
783 return true;
784 }
785 final Set<Media> media = getMedia();
786 final ContentAddition contentAddition = getPendingContentAddition();
787 if (currentState == RtpEndUserState.INCOMING_CALL) {
788 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
789 }
790 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
791 requireRtpConnection().getState())) {
792 putScreenInCallMode();
793 }
794 setWidth(currentState);
795 updateVideoViews(currentState);
796 updateStateDisplay(currentState, media, contentAddition);
797 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
798 updateButtonConfiguration(currentState, media, contentAddition);
799 updateIncomingCallScreen(currentState);
800 invalidateOptionsMenu();
801 return false;
802 }
803
804 private void initializeWithTerminatedSessionState(
805 final Account account,
806 final Jid with,
807 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
808 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
809 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
810 finish();
811 return;
812 }
813 final RtpEndUserState state = terminatedRtpSession.state;
814 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
815 updateButtonConfiguration(state);
816 updateStateDisplay(state);
817 updateIncomingCallScreen(state);
818 updateCallDuration();
819 updateVerifiedShield(false);
820 invalidateOptionsMenu();
821 setWith(account.getRoster().getContact(with), state);
822 }
823
824 private void reInitializeActivityWithRunningRtpSession(
825 final Account account, Jid with, String sessionId) {
826 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
827 resetIntent(account, with, sessionId);
828 }
829
830 private void resetIntent(final Account account, final Jid with, final String sessionId) {
831 final Intent intent = new Intent(Intent.ACTION_VIEW);
832 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
833 intent.putExtra(EXTRA_WITH, with.toEscapedString());
834 intent.putExtra(EXTRA_SESSION_ID, sessionId);
835 setIntent(intent);
836 }
837
838 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
839 surfaceViewRenderer.setVisibility(View.VISIBLE);
840 try {
841 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
842 } catch (final IllegalStateException ignored) {
843 // SurfaceViewRenderer was already initialized
844 } catch (final RuntimeException e) {
845 if (Throwables.getRootCause(e) instanceof GLException glException) {
846 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
847 }
848 }
849 surfaceViewRenderer.setEnableHardwareScaler(true);
850 }
851
852 private void updateStateDisplay(final RtpEndUserState state) {
853 updateStateDisplay(state, Collections.emptySet(), null);
854 }
855
856 private void updateStateDisplay(
857 final RtpEndUserState state,
858 final Set<Media> media,
859 final ContentAddition contentAddition) {
860 switch (state) {
861 case INCOMING_CALL -> {
862 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
863 if (media.contains(Media.VIDEO)) {
864 setTitle(R.string.rtp_state_incoming_video_call);
865 } else {
866 setTitle(R.string.rtp_state_incoming_call);
867 }
868 }
869 case INCOMING_CONTENT_ADD -> {
870 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
871 setTitle(R.string.rtp_state_content_add_video);
872 } else {
873 setTitle(R.string.rtp_state_content_add);
874 }
875 }
876 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
877 case CONNECTED -> setTitle(R.string.rtp_state_connected);
878 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
879 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
880 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
881 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
882 case RINGING -> setTitle(R.string.rtp_state_ringing);
883 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
884 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
885 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
886 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
887 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
888 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
889 case ENDED -> throw new IllegalStateException(
890 "Activity should have called finishAndReleaseWakeLock();");
891 default -> throw new IllegalStateException(
892 String.format("State %s has not been handled in UI", state));
893 }
894 }
895
896 private void updateVerifiedShield(final boolean verified) {
897 if (isPictureInPicture()) {
898 this.binding.verified.setVisibility(View.GONE);
899 return;
900 }
901 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
902 }
903
904 private void updateIncomingCallScreen(final RtpEndUserState state) {
905 updateIncomingCallScreen(state, null);
906 }
907
908 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
909 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
910 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
911 if (show) {
912 binding.contactPhoto.setVisibility(View.VISIBLE);
913 if (contact == null) {
914 AvatarWorkerTask.loadAvatar(
915 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
916 } else {
917 AvatarWorkerTask.loadAvatar(
918 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
919 }
920 } else {
921 binding.contactPhoto.setVisibility(View.GONE);
922 }
923 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
924 binding.usingAccount.setVisibility(View.VISIBLE);
925 binding.usingAccount.setText(
926 getString(
927 R.string.using_account,
928 account.getJid().asBareJid().toEscapedString()));
929 } else {
930 binding.usingAccount.setVisibility(View.GONE);
931 binding.contactPhoto.setVisibility(View.GONE);
932 }
933 }
934
935 private Set<Media> getMedia() {
936 return requireRtpConnection().getMedia();
937 }
938
939 public ContentAddition getPendingContentAddition() {
940 return requireRtpConnection().getPendingContentAddition();
941 }
942
943 private void updateButtonConfiguration(final RtpEndUserState state) {
944 updateButtonConfiguration(state, Collections.emptySet(), null);
945 }
946
947 @SuppressLint("RestrictedApi")
948 private void updateButtonConfiguration(
949 final RtpEndUserState state,
950 final Set<Media> media,
951 final ContentAddition contentAddition) {
952 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
953 this.binding.rejectCall.setVisibility(View.INVISIBLE);
954 this.binding.endCall.setVisibility(View.INVISIBLE);
955 this.binding.acceptCall.setVisibility(View.INVISIBLE);
956 } else if (state == RtpEndUserState.INCOMING_CALL) {
957 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
958 this.binding.rejectCall.setOnClickListener(this::rejectCall);
959 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
960 this.binding.rejectCall.setVisibility(View.VISIBLE);
961 this.binding.endCall.setVisibility(View.INVISIBLE);
962 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
963 this.binding.acceptCall.setOnClickListener(this::acceptCall);
964 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
965 this.binding.acceptCall.setVisibility(View.VISIBLE);
966 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
967 this.binding.rejectCall.setContentDescription(
968 getString(R.string.reject_switch_to_video));
969 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
970 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
971 this.binding.rejectCall.setVisibility(View.VISIBLE);
972 this.binding.endCall.setVisibility(View.INVISIBLE);
973 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
974 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
975 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
976 this.binding.acceptCall.setVisibility(View.VISIBLE);
977 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
978 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
979 this.binding.rejectCall.setOnClickListener(this::exit);
980 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
981 this.binding.rejectCall.setVisibility(View.VISIBLE);
982 this.binding.endCall.setVisibility(View.INVISIBLE);
983 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
984 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
985 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
986 this.binding.acceptCall.setVisibility(View.VISIBLE);
987 } else if (asList(
988 RtpEndUserState.CONNECTIVITY_ERROR,
989 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
990 RtpEndUserState.APPLICATION_ERROR,
991 RtpEndUserState.RETRACTED,
992 RtpEndUserState.SECURITY_ERROR)
993 .contains(state)) {
994 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
995 this.binding.rejectCall.setOnClickListener(this::exit);
996 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
997 this.binding.rejectCall.setVisibility(View.VISIBLE);
998 this.binding.endCall.setVisibility(View.INVISIBLE);
999 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1000 this.binding.acceptCall.setOnClickListener(this::retry);
1001 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1002 this.binding.acceptCall.setVisibility(View.VISIBLE);
1003 } else {
1004 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1005 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1006 this.binding.endCall.setOnClickListener(this::endCall);
1007 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1008 this.binding.endCall.setVisibility(View.VISIBLE);
1009 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1010 }
1011 updateInCallButtonConfiguration(state, media);
1012 }
1013
1014 private boolean isPictureInPicture() {
1015 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1016 return isInPictureInPictureMode();
1017 } else {
1018 return false;
1019 }
1020 }
1021
1022 private void updateInCallButtonConfiguration() {
1023 updateInCallButtonConfiguration(
1024 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1025 }
1026
1027 @SuppressLint("RestrictedApi")
1028 private void updateInCallButtonConfiguration(
1029 final RtpEndUserState state, final Set<Media> media) {
1030 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1031 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1032 if (media.contains(Media.VIDEO)) {
1033 final JingleRtpConnection rtpConnection = requireRtpConnection();
1034 updateInCallButtonConfigurationVideo(
1035 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1036 } else {
1037 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1038 updateInCallButtonConfigurationSpeaker(
1039 callIntegration.getSelectedAudioDevice(),
1040 callIntegration.getAudioDevices().size());
1041 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1042 }
1043 if (media.contains(Media.AUDIO)) {
1044 updateInCallButtonConfigurationMicrophone(
1045 requireRtpConnection().isMicrophoneEnabled());
1046 } else {
1047 this.binding.inCallActionLeft.setVisibility(View.GONE);
1048 }
1049 } else {
1050 this.binding.inCallActionLeft.setVisibility(View.GONE);
1051 this.binding.inCallActionRight.setVisibility(View.GONE);
1052 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1053 }
1054 }
1055
1056 @SuppressLint("RestrictedApi")
1057 private void updateInCallButtonConfigurationSpeaker(
1058 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1059 switch (selectedAudioDevice) {
1060 case EARPIECE -> {
1061 this.binding.inCallActionRight.setImageResource(
1062 R.drawable.ic_volume_off_black_24dp);
1063 if (numberOfChoices >= 2) {
1064 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1065 } else {
1066 this.binding.inCallActionRight.setOnClickListener(null);
1067 this.binding.inCallActionRight.setClickable(false);
1068 }
1069 }
1070 case WIRED_HEADSET -> {
1071 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1072 this.binding.inCallActionRight.setOnClickListener(null);
1073 this.binding.inCallActionRight.setClickable(false);
1074 }
1075 case SPEAKER_PHONE -> {
1076 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1077 if (numberOfChoices >= 2) {
1078 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1079 } else {
1080 this.binding.inCallActionRight.setOnClickListener(null);
1081 this.binding.inCallActionRight.setClickable(false);
1082 }
1083 }
1084 case BLUETOOTH -> {
1085 this.binding.inCallActionRight.setImageResource(
1086 R.drawable.ic_bluetooth_audio_black_24dp);
1087 this.binding.inCallActionRight.setOnClickListener(null);
1088 this.binding.inCallActionRight.setClickable(false);
1089 }
1090 }
1091 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1092 }
1093
1094 @SuppressLint("RestrictedApi")
1095 private void updateInCallButtonConfigurationVideo(
1096 final boolean videoEnabled, final boolean isCameraSwitchable) {
1097 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1098 if (isCameraSwitchable) {
1099 this.binding.inCallActionFarRight.setImageResource(
1100 R.drawable.ic_flip_camera_android_black_24dp);
1101 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1102 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1103 } else {
1104 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1105 }
1106 if (videoEnabled) {
1107 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1108 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1109 } else {
1110 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1111 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1112 }
1113 }
1114
1115 private void switchCamera(final View view) {
1116 Futures.addCallback(
1117 requireRtpConnection().switchCamera(),
1118 new FutureCallback<>() {
1119 @Override
1120 public void onSuccess(@Nullable Boolean isFrontCamera) {
1121 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1122 }
1123
1124 @Override
1125 public void onFailure(@NonNull final Throwable throwable) {
1126 Log.d(
1127 Config.LOGTAG,
1128 "could not switch camera",
1129 Throwables.getRootCause(throwable));
1130 Toast.makeText(
1131 RtpSessionActivity.this,
1132 R.string.could_not_switch_camera,
1133 Toast.LENGTH_LONG)
1134 .show();
1135 }
1136 },
1137 MainThreadExecutor.getInstance());
1138 }
1139
1140 private void enableVideo(View view) {
1141 try {
1142 requireRtpConnection().setVideoEnabled(true);
1143 } catch (final IllegalStateException e) {
1144 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1145 return;
1146 }
1147 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1148 }
1149
1150 private void disableVideo(View view) {
1151 final JingleRtpConnection rtpConnection = requireRtpConnection();
1152 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1153 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1154 rtpConnection.retractContentAdd();
1155 return;
1156 }
1157 requireRtpConnection().setVideoEnabled(false);
1158 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1159 }
1160
1161 @SuppressLint("RestrictedApi")
1162 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1163 if (microphoneEnabled) {
1164 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1165 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1166 } else {
1167 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1168 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1169 }
1170 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1171 }
1172
1173 private void updateCallDuration() {
1174 final JingleRtpConnection connection =
1175 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1176 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1177 this.binding.duration.setVisibility(View.GONE);
1178 return;
1179 }
1180 if (connection.zeroDuration()) {
1181 this.binding.duration.setVisibility(View.GONE);
1182 } else {
1183 this.binding.duration.setText(
1184 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1185 this.binding.duration.setVisibility(View.VISIBLE);
1186 }
1187 }
1188
1189 private void updateVideoViews(final RtpEndUserState state) {
1190 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1191 binding.localVideo.setVisibility(View.GONE);
1192 binding.localVideo.release();
1193 binding.remoteVideoWrapper.setVisibility(View.GONE);
1194 binding.remoteVideo.release();
1195 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1196 if (isPictureInPicture()) {
1197 binding.appBarLayout.setVisibility(View.GONE);
1198 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1199 if (Arrays.asList(
1200 RtpEndUserState.APPLICATION_ERROR,
1201 RtpEndUserState.CONNECTIVITY_ERROR,
1202 RtpEndUserState.SECURITY_ERROR)
1203 .contains(state)) {
1204 binding.pipWarning.setVisibility(View.VISIBLE);
1205 binding.pipWaiting.setVisibility(View.GONE);
1206 } else {
1207 binding.pipWarning.setVisibility(View.GONE);
1208 binding.pipWaiting.setVisibility(View.GONE);
1209 }
1210 } else {
1211 binding.appBarLayout.setVisibility(View.VISIBLE);
1212 binding.pipPlaceholder.setVisibility(View.GONE);
1213 }
1214 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1215 return;
1216 }
1217 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1218 binding.localVideo.setVisibility(View.GONE);
1219 binding.remoteVideoWrapper.setVisibility(View.GONE);
1220 binding.appBarLayout.setVisibility(View.GONE);
1221 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1222 binding.pipWarning.setVisibility(View.GONE);
1223 binding.pipWaiting.setVisibility(View.VISIBLE);
1224 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1225 return;
1226 }
1227 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1228 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1229 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1230 // paint local view over remote view
1231 binding.localVideo.setZOrderMediaOverlay(true);
1232 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1233 addSink(localVideoTrack.get(), binding.localVideo);
1234 } else {
1235 binding.localVideo.setVisibility(View.GONE);
1236 }
1237 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1238 if (remoteVideoTrack.isPresent()) {
1239 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1240 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1241 binding.remoteVideo.setScalingType(
1242 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1243 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1244 if (state == RtpEndUserState.CONNECTED) {
1245 binding.appBarLayout.setVisibility(View.GONE);
1246 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1247 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1248 } else {
1249 binding.appBarLayout.setVisibility(View.VISIBLE);
1250 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1251 binding.remoteVideoWrapper.setVisibility(View.GONE);
1252 }
1253 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1254 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1255 } else {
1256 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1257 }
1258 } else {
1259 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1260 binding.remoteVideoWrapper.setVisibility(View.GONE);
1261 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1262 }
1263 }
1264
1265 private Optional<VideoTrack> getLocalVideoTrack() {
1266 final JingleRtpConnection connection =
1267 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1268 if (connection == null) {
1269 return Optional.absent();
1270 }
1271 return connection.getLocalVideoTrack();
1272 }
1273
1274 private Optional<VideoTrack> getRemoteVideoTrack() {
1275 final JingleRtpConnection connection =
1276 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1277 if (connection == null) {
1278 return Optional.absent();
1279 }
1280 return connection.getRemoteVideoTrack();
1281 }
1282
1283 private void disableMicrophone(View view) {
1284 final JingleRtpConnection rtpConnection = requireRtpConnection();
1285 if (rtpConnection.setMicrophoneEnabled(false)) {
1286 updateInCallButtonConfiguration();
1287 }
1288 }
1289
1290 private void enableMicrophone(View view) {
1291 final JingleRtpConnection rtpConnection = requireRtpConnection();
1292 if (rtpConnection.setMicrophoneEnabled(true)) {
1293 updateInCallButtonConfiguration();
1294 }
1295 }
1296
1297 private void switchToEarpiece(View view) {
1298 requireRtpConnection()
1299 .getCallIntegration()
1300 .setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1301 acquireProximityWakeLock();
1302 }
1303
1304 private void switchToSpeaker(View view) {
1305 requireRtpConnection()
1306 .getCallIntegration()
1307 .setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1308 releaseProximityWakeLock();
1309 }
1310
1311 private void retry(final View view) {
1312 final Intent intent = getIntent();
1313 final Account account = extractAccount(intent);
1314 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1315 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1316 final String action = intent.getAction();
1317 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1318 this.rtpConnectionReference = null;
1319 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1320 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1321 }
1322
1323 private void exit(final View view) {
1324 finish();
1325 }
1326
1327 private void recordVoiceMail(final View view) {
1328 final Intent intent = getIntent();
1329 final Account account = extractAccount(intent);
1330 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1331 final Conversation conversation =
1332 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1333 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1334 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1335 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1336 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1337 launchIntent.putExtra(
1338 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1339 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1340 startActivity(launchIntent);
1341 finish();
1342 }
1343
1344 private Contact getWith() {
1345 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1346 final Account account = id.account;
1347 return account.getRoster().getContact(id.with);
1348 }
1349
1350 private JingleRtpConnection requireRtpConnection() {
1351 final JingleRtpConnection connection =
1352 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1353 if (connection == null) {
1354 throw new IllegalStateException("No RTP connection found");
1355 }
1356 return connection;
1357 }
1358
1359 @Override
1360 public void onJingleRtpConnectionUpdate(
1361 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1362 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1363 if (END_CARD.contains(state)) {
1364 Log.d(Config.LOGTAG, "end card reached");
1365 releaseProximityWakeLock();
1366 runOnUiThread(
1367 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1368 }
1369 if (with.isBareJid()) {
1370 updateRtpSessionProposalState(account, with, state);
1371 return;
1372 }
1373 if (emptyReference(this.rtpConnectionReference)) {
1374 if (END_CARD.contains(state)) {
1375 Log.d(Config.LOGTAG, "not reinitializing session");
1376 return;
1377 }
1378 // this happens when going from proposed session to actual session
1379 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1380 return;
1381 }
1382 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1383 final boolean verified = requireRtpConnection().isVerified();
1384 final Set<Media> media = getMedia();
1385 lockOrientation(media);
1386 final ContentAddition contentAddition = getPendingContentAddition();
1387 final Contact contact = getWith();
1388 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1389 if (state == RtpEndUserState.ENDED) {
1390 finish();
1391 return;
1392 }
1393 runOnUiThread(
1394 () -> {
1395 updateStateDisplay(state, media, contentAddition);
1396 updateVerifiedShield(
1397 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1398 updateButtonConfiguration(state, media, contentAddition);
1399 updateVideoViews(state);
1400 updateIncomingCallScreen(state, contact);
1401 invalidateOptionsMenu();
1402 });
1403 if (END_CARD.contains(state)) {
1404 final JingleRtpConnection rtpConnection = requireRtpConnection();
1405 resetIntent(account, with, state, rtpConnection.getMedia());
1406 releaseVideoTracks(rtpConnection);
1407 this.rtpConnectionReference = null;
1408 }
1409 } else {
1410 Log.d(Config.LOGTAG, "received update for other rtp session");
1411 }
1412 }
1413
1414 @Override
1415 public void onAudioDeviceChanged(
1416 final CallIntegration.AudioDevice selectedAudioDevice,
1417 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1418 Log.d(
1419 Config.LOGTAG,
1420 "onAudioDeviceChanged in activity: selected:"
1421 + selectedAudioDevice
1422 + ", available:"
1423 + availableAudioDevices);
1424 try {
1425 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1426 final Set<Media> media = getMedia();
1427 if (END_CARD.contains(endUserState)) {
1428 Log.d(
1429 Config.LOGTAG,
1430 "onAudioDeviceChanged() nothing to do because end card has been reached");
1431 } else {
1432 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1433 final CallIntegration callIntegration =
1434 requireRtpConnection().getCallIntegration();
1435 updateInCallButtonConfigurationSpeaker(
1436 callIntegration.getSelectedAudioDevice(),
1437 callIntegration.getAudioDevices().size());
1438 }
1439 Log.d(
1440 Config.LOGTAG,
1441 "put proximity wake lock into proper state after device update");
1442 putProximityWakeLockInProperState(selectedAudioDevice);
1443 }
1444 } catch (final IllegalStateException e) {
1445 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1446 }
1447 }
1448
1449 private void updateRtpSessionProposalState(
1450 final Account account, final Jid with, final RtpEndUserState state) {
1451 final Intent currentIntent = getIntent();
1452 final String withExtra =
1453 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1454 if (withExtra == null) {
1455 return;
1456 }
1457 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1458 runOnUiThread(
1459 () -> {
1460 updateVerifiedShield(false);
1461 updateStateDisplay(state);
1462 updateButtonConfiguration(state);
1463 updateIncomingCallScreen(state);
1464 invalidateOptionsMenu();
1465 });
1466 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1467 }
1468 }
1469
1470 private void resetIntent(final Bundle extras) {
1471 final Intent intent = new Intent(Intent.ACTION_VIEW);
1472 intent.putExtras(extras);
1473 setIntent(intent);
1474 }
1475
1476 private void resetIntent(
1477 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1478 final Intent intent = new Intent(Intent.ACTION_VIEW);
1479 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1480 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1481 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1482 } else {
1483 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1484 }
1485 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1486 intent.putExtra(
1487 EXTRA_LAST_ACTION,
1488 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1489 setIntent(intent);
1490 }
1491
1492 private static boolean emptyReference(final WeakReference<?> weakReference) {
1493 return weakReference == null || weakReference.get() == null;
1494 }
1495}