1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.PictureInPictureParams;
9import android.content.ActivityNotFoundException;
10import android.content.Context;
11import android.content.Intent;
12import android.content.pm.PackageManager;
13import android.os.Build;
14import android.os.Bundle;
15import android.os.Handler;
16import android.os.PowerManager;
17import android.os.SystemClock;
18import android.util.Log;
19import android.util.Rational;
20import android.view.KeyEvent;
21import android.view.Menu;
22import android.view.MenuItem;
23import android.view.View;
24import android.view.WindowManager;
25import android.widget.Toast;
26
27import androidx.annotation.NonNull;
28import androidx.annotation.RequiresApi;
29import androidx.annotation.StringRes;
30import androidx.databinding.DataBindingUtil;
31
32import com.google.common.base.Optional;
33import com.google.common.base.Preconditions;
34import com.google.common.base.Throwables;
35import com.google.common.collect.ImmutableList;
36import com.google.common.collect.ImmutableSet;
37import com.google.common.util.concurrent.FutureCallback;
38import com.google.common.util.concurrent.Futures;
39
40import org.checkerframework.checker.nullness.compatqual.NullableDecl;
41import org.jetbrains.annotations.NotNull;
42import org.webrtc.RendererCommon;
43import org.webrtc.SurfaceViewRenderer;
44import org.webrtc.VideoTrack;
45
46import java.lang.ref.WeakReference;
47import java.util.Arrays;
48import java.util.Collections;
49import java.util.List;
50import java.util.Set;
51
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.R;
54import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.services.AppRTCAudioManager;
59import eu.siacs.conversations.services.XmppConnectionService;
60import eu.siacs.conversations.ui.widget.DialpadView;
61import eu.siacs.conversations.ui.util.AvatarWorkerTask;
62import eu.siacs.conversations.ui.util.MainThreadExecutor;
63import eu.siacs.conversations.ui.util.Rationals;
64import eu.siacs.conversations.utils.PermissionUtils;
65import eu.siacs.conversations.utils.TimeFrameUtils;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
69import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
70import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
71import eu.siacs.conversations.xmpp.jingle.Media;
72import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
73
74public class RtpSessionActivity extends XmppActivity
75 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
76 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
77
78 public static final String EXTRA_WITH = "with";
79 public static final String EXTRA_SESSION_ID = "session_id";
80 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
81 public static final String EXTRA_LAST_ACTION = "last_action";
82 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
83 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
84 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
85
86 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
87
88 public static final List<RtpEndUserState> END_CARD =
89 Arrays.asList(
90 RtpEndUserState.APPLICATION_ERROR,
91 RtpEndUserState.SECURITY_ERROR,
92 RtpEndUserState.DECLINED_OR_BUSY,
93 RtpEndUserState.CONNECTIVITY_ERROR,
94 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
95 RtpEndUserState.RETRACTED);
96 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
97 Arrays.asList(
98 RtpEndUserState.APPLICATION_ERROR,
99 RtpEndUserState.CONNECTIVITY_ERROR,
100 RtpEndUserState.SECURITY_ERROR);
101 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
102 Arrays.asList(
103 RtpEndUserState.CONNECTING,
104 RtpEndUserState.CONNECTED,
105 RtpEndUserState.RECONNECTING);
106 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
107 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
108 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
109 Arrays.asList(
110 RtpEndUserState.ACCEPTING_CALL,
111 RtpEndUserState.CONNECTING,
112 RtpEndUserState.RECONNECTING);
113 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
114 private static final int REQUEST_ACCEPT_CALL = 0x1111;
115 private WeakReference<JingleRtpConnection> rtpConnectionReference;
116
117 private ActivityRtpSessionBinding binding;
118 private PowerManager.WakeLock mProximityWakeLock;
119
120 private final Handler mHandler = new Handler();
121 private final Runnable mTickExecutor =
122 new Runnable() {
123 @Override
124 public void run() {
125 updateCallDuration();
126 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
127 }
128 };
129
130 private static Set<Media> actionToMedia(final String action) {
131 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
132 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
133 } else {
134 return ImmutableSet.of(Media.AUDIO);
135 }
136 }
137
138 private static void addSink(
139 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
140 try {
141 videoTrack.addSink(surfaceViewRenderer);
142 } catch (final IllegalStateException e) {
143 Log.e(
144 Config.LOGTAG,
145 "possible race condition on trying to display video track. ignoring",
146 e);
147 }
148 }
149
150 @Override
151 public void onCreate(Bundle savedInstanceState) {
152 super.onCreate(savedInstanceState);
153 getWindow()
154 .addFlags(
155 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
156 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
157 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
158 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
159 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
160 setSupportActionBar(binding.toolbar);
161
162 binding.dialpad.setClickConsumer(tag -> {
163 requireRtpConnection().applyDtmfTone(tag);
164 });
165
166 if (savedInstanceState != null) {
167 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
168 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
169 }
170 }
171
172 @Override
173 public boolean onCreateOptionsMenu(final Menu menu) {
174 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
175 final MenuItem help = menu.findItem(R.id.action_help);
176 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
177 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
178 help.setVisible(isHelpButtonVisible());
179 gotoChat.setVisible(isSwitchToConversationVisible());
180 dialpad.setVisible(isAudioOnlyConversation());
181 return super.onCreateOptionsMenu(menu);
182 }
183
184 @Override
185 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
186 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
187 if (xmppConnectionService != null) {
188 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
189 return true;
190 }
191 }
192 }
193 return super.onKeyDown(keyCode, event);
194 }
195
196 private boolean isHelpButtonVisible() {
197 try {
198 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
199 } catch (IllegalStateException e) {
200 final Intent intent = getIntent();
201 final String state =
202 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
203 if (state != null) {
204 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
205 } else {
206 return false;
207 }
208 }
209 }
210
211 private boolean isSwitchToConversationVisible() {
212 final JingleRtpConnection connection =
213 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
214 return connection != null
215 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
216 }
217
218 private boolean isAudioOnlyConversation() {
219 final JingleRtpConnection connection =
220 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
221
222 return connection != null && !connection.getMedia().contains(Media.VIDEO);
223 }
224
225 private void switchToConversation() {
226 final Contact contact = getWith();
227 final Conversation conversation =
228 xmppConnectionService.findOrCreateConversation(
229 contact.getAccount(), contact.getJid(), false, true);
230 switchToConversation(conversation);
231 }
232
233 private void toggleDialpadVisibility() {
234 if (binding.dialpad.getVisibility() == View.VISIBLE) {
235 binding.dialpad.setVisibility(View.GONE);
236 }
237 else {
238 binding.dialpad.setVisibility(View.VISIBLE);
239 }
240 }
241
242 public boolean onOptionsItemSelected(final MenuItem item) {
243 switch (item.getItemId()) {
244 case R.id.action_help:
245 launchHelpInBrowser();
246 break;
247 case R.id.action_goto_chat:
248 switchToConversation();
249 break;
250 case R.id.action_dialpad:
251 toggleDialpadVisibility();
252 break;
253 }
254 return super.onOptionsItemSelected(item);
255 }
256
257 private void launchHelpInBrowser() {
258 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
259 try {
260 startActivity(intent);
261 } catch (final ActivityNotFoundException e) {
262 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
263 .show();
264 }
265 }
266
267 private void endCall(View view) {
268 endCall();
269 }
270
271 private void endCall() {
272 if (this.rtpConnectionReference == null) {
273 retractSessionProposal();
274 finish();
275 } else {
276 requireRtpConnection().endCall();
277 }
278 }
279
280 private void retractSessionProposal() {
281 final Intent intent = getIntent();
282 final String action = intent.getAction();
283 final Account account = extractAccount(intent);
284 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
285 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
286 if (!Intent.ACTION_VIEW.equals(action)
287 || state == null
288 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
289 resetIntent(
290 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
291 }
292 xmppConnectionService
293 .getJingleConnectionManager()
294 .retractSessionProposal(account, with.asBareJid());
295 }
296
297 private void rejectCall(View view) {
298 requireRtpConnection().rejectCall();
299 finish();
300 }
301
302 private void acceptCall(View view) {
303 requestPermissionsAndAcceptCall();
304 }
305
306 private void requestPermissionsAndAcceptCall() {
307 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
308 if (getMedia().contains(Media.VIDEO)) {
309 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
310 } else {
311 permissions.add(Manifest.permission.RECORD_AUDIO);
312 }
313 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
314 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
315 }
316 if (PermissionUtils.hasPermission(this, permissions.build(), REQUEST_ACCEPT_CALL)) {
317 putScreenInCallMode();
318 checkRecorderAndAcceptCall();
319 }
320 }
321
322 private void checkRecorderAndAcceptCall() {
323 checkMicrophoneAvailabilityAsync();
324 try {
325 requireRtpConnection().acceptCall();
326 } catch (final IllegalStateException e) {
327 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
328 }
329 }
330
331 private void checkMicrophoneAvailabilityAsync() {
332 new Thread(this::checkMicrophoneAvailability).start();
333 }
334
335 private void checkMicrophoneAvailability() {
336 final long start = SystemClock.elapsedRealtime();
337 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
338 final long stop = SystemClock.elapsedRealtime();
339 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
340 if (isMicrophoneAvailable) {
341 return;
342 }
343 runOnUiThread(
344 () ->
345 Toast.makeText(this, R.string.microphone_unavailable, Toast.LENGTH_LONG)
346 .show());
347 }
348
349 private void putScreenInCallMode() {
350 putScreenInCallMode(requireRtpConnection().getMedia());
351 }
352
353 private void putScreenInCallMode(final Set<Media> media) {
354 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
355 if (!media.contains(Media.VIDEO)) {
356 final JingleRtpConnection rtpConnection =
357 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
358 final AppRTCAudioManager audioManager =
359 rtpConnection == null ? null : rtpConnection.getAudioManager();
360 if (audioManager == null
361 || audioManager.getSelectedAudioDevice()
362 == AppRTCAudioManager.AudioDevice.EARPIECE) {
363 acquireProximityWakeLock();
364 }
365 }
366 }
367
368 @SuppressLint("WakelockTimeout")
369 private void acquireProximityWakeLock() {
370 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
371 if (powerManager == null) {
372 Log.e(Config.LOGTAG, "power manager not available");
373 return;
374 }
375 if (isFinishing()) {
376 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
377 return;
378 }
379 if (this.mProximityWakeLock == null) {
380 this.mProximityWakeLock =
381 powerManager.newWakeLock(
382 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
383 }
384 if (!this.mProximityWakeLock.isHeld()) {
385 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
386 this.mProximityWakeLock.acquire();
387 }
388 }
389
390 private void releaseProximityWakeLock() {
391 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
392 Log.d(Config.LOGTAG, "releasing proximity wake lock");
393 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
394 this.mProximityWakeLock = null;
395 }
396 }
397
398 private void putProximityWakeLockInProperState(
399 final AppRTCAudioManager.AudioDevice audioDevice) {
400 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
401 acquireProximityWakeLock();
402 } else {
403 releaseProximityWakeLock();
404 }
405 }
406
407 @Override
408 protected void refreshUiReal() {}
409
410 @Override
411 public void onNewIntent(final Intent intent) {
412 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
413 super.onNewIntent(intent);
414 setIntent(intent);
415 if (xmppConnectionService == null) {
416 Log.d(
417 Config.LOGTAG,
418 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
419 return;
420 }
421 final Account account = extractAccount(intent);
422 final String action = intent.getAction();
423 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
424 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
425 if (sessionId != null) {
426 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
427 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
428 return;
429 }
430 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
431 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
432 requestPermissionsAndAcceptCall();
433 resetIntent(intent.getExtras());
434 }
435 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
436 proposeJingleRtpSession(account, with, actionToMedia(action));
437 setWith(account.getRoster().getContact(with), null);
438 } else {
439 throw new IllegalStateException("received onNewIntent without sessionId");
440 }
441 }
442
443 @Override
444 void onBackendConnected() {
445 final Intent intent = getIntent();
446 final String action = intent.getAction();
447 final Account account = extractAccount(intent);
448 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
449 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
450 if (sessionId != null) {
451 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
452 return;
453 }
454 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
455 Log.d(Config.LOGTAG, "intent action was accept");
456 requestPermissionsAndAcceptCall();
457 resetIntent(intent.getExtras());
458 }
459 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
460 proposeJingleRtpSession(account, with, actionToMedia(action));
461 setWith(account.getRoster().getContact(with), null);
462 } else if (Intent.ACTION_VIEW.equals(action)) {
463 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
464 final RtpEndUserState state =
465 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
466 if (state != null) {
467 Log.d(Config.LOGTAG, "restored last state from intent extra");
468 updateButtonConfiguration(state);
469 updateVerifiedShield(false);
470 updateStateDisplay(state);
471 updateIncomingCallScreen(state);
472 invalidateOptionsMenu();
473 }
474 setWith(account.getRoster().getContact(with), state);
475 if (xmppConnectionService
476 .getJingleConnectionManager()
477 .fireJingleRtpConnectionStateUpdates()) {
478 return;
479 }
480 if (END_CARD.contains(state)
481 || xmppConnectionService
482 .getJingleConnectionManager()
483 .hasMatchingProposal(account, with)) {
484 return;
485 }
486 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
487 finish();
488 }
489 }
490
491 private void setWith(final RtpEndUserState state) {
492 setWith(getWith(), state);
493 }
494
495 private void setWith(final Contact contact, final RtpEndUserState state) {
496 binding.with.setText(contact.getDisplayName());
497 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
498 .contains(state)) {
499 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
500 binding.withJid.setVisibility(View.VISIBLE);
501 } else {
502 binding.withJid.setVisibility(View.GONE);
503 }
504 }
505
506 private void proposeJingleRtpSession(
507 final Account account, final Jid with, final Set<Media> media) {
508 checkMicrophoneAvailabilityAsync();
509 if (with.isBareJid()) {
510 xmppConnectionService
511 .getJingleConnectionManager()
512 .proposeJingleRtpSession(account, with, media);
513 } else {
514 final String sessionId =
515 xmppConnectionService
516 .getJingleConnectionManager()
517 .initializeRtpSession(account, with, media);
518 initializeActivityWithRunningRtpSession(account, with, sessionId);
519 resetIntent(account, with, sessionId);
520 }
521 putScreenInCallMode(media);
522 }
523
524 @Override
525 public void onRequestPermissionsResult(
526 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
527 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
528 final PermissionUtils.PermissionResult permissionResult =
529 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
530 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
531 if (requestCode == REQUEST_ACCEPT_CALL) {
532 checkRecorderAndAcceptCall();
533 }
534 } else {
535 @StringRes int res;
536 final String firstDenied =
537 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
538 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
539 res = R.string.no_microphone_permission;
540 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
541 res = R.string.no_camera_permission;
542 } else {
543 throw new IllegalStateException("Invalid permission result request");
544 }
545 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
546 .show();
547 }
548 }
549
550 @Override
551 public void onStart() {
552 super.onStart();
553 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
554 this.binding.remoteVideo.setOnAspectRatioChanged(this);
555 }
556
557 @Override
558 public void onStop() {
559 mHandler.removeCallbacks(mTickExecutor);
560 binding.remoteVideo.release();
561 binding.remoteVideo.setOnAspectRatioChanged(null);
562 binding.localVideo.release();
563 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
564 final JingleRtpConnection jingleRtpConnection =
565 weakReference == null ? null : weakReference.get();
566 if (jingleRtpConnection != null) {
567 releaseVideoTracks(jingleRtpConnection);
568 }
569 releaseProximityWakeLock();
570 super.onStop();
571 }
572
573 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
574 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
575 if (remoteVideo.isPresent()) {
576 remoteVideo.get().removeSink(binding.remoteVideo);
577 }
578 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
579 if (localVideo.isPresent()) {
580 localVideo.get().removeSink(binding.localVideo);
581 }
582 }
583
584 @Override
585 public void onBackPressed() {
586 if (isConnected()) {
587 if (switchToPictureInPicture()) {
588 return;
589 }
590 } else {
591 endCall();
592 }
593 super.onBackPressed();
594 }
595
596 @Override
597 public void onUserLeaveHint() {
598 super.onUserLeaveHint();
599 if (switchToPictureInPicture()) {
600 return;
601 }
602 // TODO apparently this method is not getting called on Android 10 when using the task
603 // switcher
604 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
605 retractSessionProposal();
606 }
607 }
608
609 private boolean isConnected() {
610 final JingleRtpConnection connection =
611 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
612 return connection != null
613 && STATES_CONSIDERED_CONNECTED.contains(connection.getEndUserState());
614 }
615
616 private boolean switchToPictureInPicture() {
617 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
618 if (shouldBePictureInPicture()) {
619 startPictureInPicture();
620 return true;
621 }
622 }
623 return false;
624 }
625
626 @RequiresApi(api = Build.VERSION_CODES.O)
627 private void startPictureInPicture() {
628 try {
629 final Rational rational = this.binding.remoteVideo.getAspectRatio();
630 final Rational clippedRational = Rationals.clip(rational);
631 Log.d(
632 Config.LOGTAG,
633 "suggested rational " + rational + ". clipped to " + clippedRational);
634 enterPictureInPictureMode(
635 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
636 } catch (final IllegalStateException e) {
637 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
638 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
639 }
640 }
641
642 @Override
643 public void onAspectRatioChanged(final Rational rational) {
644 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
645 final Rational clippedRational = Rationals.clip(rational);
646 Log.d(
647 Config.LOGTAG,
648 "suggested rational after aspect ratio change "
649 + rational
650 + ". clipped to "
651 + clippedRational);
652 setPictureInPictureParams(
653 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
654 }
655 }
656
657 private boolean deviceSupportsPictureInPicture() {
658 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
659 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
660 } else {
661 return false;
662 }
663 }
664
665 private boolean shouldBePictureInPicture() {
666 try {
667 final JingleRtpConnection rtpConnection = requireRtpConnection();
668 return rtpConnection.getMedia().contains(Media.VIDEO)
669 && Arrays.asList(
670 RtpEndUserState.ACCEPTING_CALL,
671 RtpEndUserState.CONNECTING,
672 RtpEndUserState.CONNECTED)
673 .contains(rtpConnection.getEndUserState());
674 } catch (final IllegalStateException e) {
675 return false;
676 }
677 }
678
679 private boolean initializeActivityWithRunningRtpSession(
680 final Account account, Jid with, String sessionId) {
681 final WeakReference<JingleRtpConnection> reference =
682 xmppConnectionService
683 .getJingleConnectionManager()
684 .findJingleRtpConnection(account, with, sessionId);
685 if (reference == null || reference.get() == null) {
686 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
687 xmppConnectionService
688 .getJingleConnectionManager()
689 .getTerminalSessionState(with, sessionId);
690 if (terminatedRtpSession == null) {
691 throw new IllegalStateException(
692 "failed to initialize activity with running rtp session. session not found");
693 }
694 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
695 return true;
696 }
697 this.rtpConnectionReference = reference;
698 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
699 final boolean verified = requireRtpConnection().isVerified();
700 if (currentState == RtpEndUserState.ENDED) {
701 finish();
702 return true;
703 }
704 final Set<Media> media = getMedia();
705 if (currentState == RtpEndUserState.INCOMING_CALL) {
706 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
707 }
708 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
709 requireRtpConnection().getState())) {
710 putScreenInCallMode();
711 }
712 setWith(currentState);
713 updateVideoViews(currentState);
714 updateStateDisplay(currentState, media);
715 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
716 updateButtonConfiguration(currentState, media);
717 updateIncomingCallScreen(currentState);
718 invalidateOptionsMenu();
719 return false;
720 }
721
722 private void initializeWithTerminatedSessionState(
723 final Account account,
724 final Jid with,
725 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
726 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
727 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
728 finish();
729 return;
730 }
731 final RtpEndUserState state = terminatedRtpSession.state;
732 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
733 updateButtonConfiguration(state);
734 updateStateDisplay(state);
735 updateIncomingCallScreen(state);
736 updateCallDuration();
737 updateVerifiedShield(false);
738 invalidateOptionsMenu();
739 setWith(account.getRoster().getContact(with), state);
740 }
741
742 private void reInitializeActivityWithRunningRtpSession(
743 final Account account, Jid with, String sessionId) {
744 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
745 resetIntent(account, with, sessionId);
746 }
747
748 private void resetIntent(final Account account, final Jid with, final String sessionId) {
749 final Intent intent = new Intent(Intent.ACTION_VIEW);
750 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
751 intent.putExtra(EXTRA_WITH, with.toEscapedString());
752 intent.putExtra(EXTRA_SESSION_ID, sessionId);
753 setIntent(intent);
754 }
755
756 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
757 surfaceViewRenderer.setVisibility(View.VISIBLE);
758 try {
759 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
760 } catch (final IllegalStateException e) {
761 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
762 }
763 surfaceViewRenderer.setEnableHardwareScaler(true);
764 }
765
766 private void updateStateDisplay(final RtpEndUserState state) {
767 updateStateDisplay(state, Collections.emptySet());
768 }
769
770 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media) {
771 switch (state) {
772 case INCOMING_CALL:
773 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
774 if (media.contains(Media.VIDEO)) {
775 setTitle(R.string.rtp_state_incoming_video_call);
776 } else {
777 setTitle(R.string.rtp_state_incoming_call);
778 }
779 break;
780 case CONNECTING:
781 setTitle(R.string.rtp_state_connecting);
782 break;
783 case CONNECTED:
784 setTitle(R.string.rtp_state_connected);
785 break;
786 case RECONNECTING:
787 setTitle(R.string.rtp_state_reconnecting);
788 break;
789 case ACCEPTING_CALL:
790 setTitle(R.string.rtp_state_accepting_call);
791 break;
792 case ENDING_CALL:
793 setTitle(R.string.rtp_state_ending_call);
794 break;
795 case FINDING_DEVICE:
796 setTitle(R.string.rtp_state_finding_device);
797 break;
798 case RINGING:
799 setTitle(R.string.rtp_state_ringing);
800 break;
801 case DECLINED_OR_BUSY:
802 setTitle(R.string.rtp_state_declined_or_busy);
803 break;
804 case CONNECTIVITY_ERROR:
805 setTitle(R.string.rtp_state_connectivity_error);
806 break;
807 case CONNECTIVITY_LOST_ERROR:
808 setTitle(R.string.rtp_state_connectivity_lost_error);
809 break;
810 case RETRACTED:
811 setTitle(R.string.rtp_state_retracted);
812 break;
813 case APPLICATION_ERROR:
814 setTitle(R.string.rtp_state_application_failure);
815 break;
816 case SECURITY_ERROR:
817 setTitle(R.string.rtp_state_security_error);
818 break;
819 case ENDED:
820 throw new IllegalStateException(
821 "Activity should have called finishAndReleaseWakeLock();");
822 default:
823 throw new IllegalStateException(
824 String.format("State %s has not been handled in UI", state));
825 }
826 }
827
828 private void updateVerifiedShield(final boolean verified) {
829 if (isPictureInPicture()) {
830 this.binding.verified.setVisibility(View.GONE);
831 return;
832 }
833 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
834 }
835
836 private void updateIncomingCallScreen(final RtpEndUserState state) {
837 updateIncomingCallScreen(state, null);
838 }
839
840 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
841 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
842 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
843 if (show) {
844 binding.contactPhoto.setVisibility(View.VISIBLE);
845 if (contact == null) {
846 AvatarWorkerTask.loadAvatar(
847 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
848 } else {
849 AvatarWorkerTask.loadAvatar(
850 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
851 }
852 } else {
853 binding.contactPhoto.setVisibility(View.GONE);
854 }
855 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
856 binding.usingAccount.setVisibility(View.VISIBLE);
857 binding.usingAccount.setText(
858 getString(
859 R.string.using_account,
860 account.getJid().asBareJid().toEscapedString()));
861 } else {
862 binding.usingAccount.setVisibility(View.GONE);
863 binding.contactPhoto.setVisibility(View.GONE);
864 }
865 }
866
867 private Set<Media> getMedia() {
868 return requireRtpConnection().getMedia();
869 }
870
871 private void updateButtonConfiguration(final RtpEndUserState state) {
872 updateButtonConfiguration(state, Collections.emptySet());
873 }
874
875 @SuppressLint("RestrictedApi")
876 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
877 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
878 this.binding.rejectCall.setVisibility(View.INVISIBLE);
879 this.binding.endCall.setVisibility(View.INVISIBLE);
880 this.binding.acceptCall.setVisibility(View.INVISIBLE);
881 } else if (state == RtpEndUserState.INCOMING_CALL) {
882 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
883 this.binding.rejectCall.setOnClickListener(this::rejectCall);
884 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
885 this.binding.rejectCall.setVisibility(View.VISIBLE);
886 this.binding.endCall.setVisibility(View.INVISIBLE);
887 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
888 this.binding.acceptCall.setOnClickListener(this::acceptCall);
889 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
890 this.binding.acceptCall.setVisibility(View.VISIBLE);
891 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
892 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
893 this.binding.rejectCall.setOnClickListener(this::exit);
894 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
895 this.binding.rejectCall.setVisibility(View.VISIBLE);
896 this.binding.endCall.setVisibility(View.INVISIBLE);
897 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
898 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
899 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
900 this.binding.acceptCall.setVisibility(View.VISIBLE);
901 } else if (asList(
902 RtpEndUserState.CONNECTIVITY_ERROR,
903 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
904 RtpEndUserState.APPLICATION_ERROR,
905 RtpEndUserState.RETRACTED,
906 RtpEndUserState.SECURITY_ERROR)
907 .contains(state)) {
908 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
909 this.binding.rejectCall.setOnClickListener(this::exit);
910 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
911 this.binding.rejectCall.setVisibility(View.VISIBLE);
912 this.binding.endCall.setVisibility(View.INVISIBLE);
913 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
914 this.binding.acceptCall.setOnClickListener(this::retry);
915 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
916 this.binding.acceptCall.setVisibility(View.VISIBLE);
917 } else {
918 this.binding.rejectCall.setVisibility(View.INVISIBLE);
919 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
920 this.binding.endCall.setOnClickListener(this::endCall);
921 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
922 this.binding.endCall.setVisibility(View.VISIBLE);
923 this.binding.acceptCall.setVisibility(View.INVISIBLE);
924 }
925 updateInCallButtonConfiguration(state, media);
926 }
927
928 private boolean isPictureInPicture() {
929 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
930 return isInPictureInPictureMode();
931 } else {
932 return false;
933 }
934 }
935
936 private void updateInCallButtonConfiguration() {
937 updateInCallButtonConfiguration(
938 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
939 }
940
941 @SuppressLint("RestrictedApi")
942 private void updateInCallButtonConfiguration(
943 final RtpEndUserState state, final Set<Media> media) {
944 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
945 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
946 if (media.contains(Media.VIDEO)) {
947 final JingleRtpConnection rtpConnection = requireRtpConnection();
948 updateInCallButtonConfigurationVideo(
949 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
950 } else {
951 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
952 updateInCallButtonConfigurationSpeaker(
953 audioManager.getSelectedAudioDevice(),
954 audioManager.getAudioDevices().size());
955 this.binding.inCallActionFarRight.setVisibility(View.GONE);
956 }
957 if (media.contains(Media.AUDIO)) {
958 updateInCallButtonConfigurationMicrophone(
959 requireRtpConnection().isMicrophoneEnabled());
960 } else {
961 this.binding.inCallActionLeft.setVisibility(View.GONE);
962 }
963 } else {
964 this.binding.inCallActionLeft.setVisibility(View.GONE);
965 this.binding.inCallActionRight.setVisibility(View.GONE);
966 this.binding.inCallActionFarRight.setVisibility(View.GONE);
967 }
968 }
969
970 @SuppressLint("RestrictedApi")
971 private void updateInCallButtonConfigurationSpeaker(
972 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
973 switch (selectedAudioDevice) {
974 case EARPIECE:
975 this.binding.inCallActionRight.setImageResource(
976 R.drawable.ic_volume_off_black_24dp);
977 if (numberOfChoices >= 2) {
978 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
979 } else {
980 this.binding.inCallActionRight.setOnClickListener(null);
981 this.binding.inCallActionRight.setClickable(false);
982 }
983 break;
984 case WIRED_HEADSET:
985 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
986 this.binding.inCallActionRight.setOnClickListener(null);
987 this.binding.inCallActionRight.setClickable(false);
988 break;
989 case SPEAKER_PHONE:
990 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
991 if (numberOfChoices >= 2) {
992 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
993 } else {
994 this.binding.inCallActionRight.setOnClickListener(null);
995 this.binding.inCallActionRight.setClickable(false);
996 }
997 break;
998 case BLUETOOTH:
999 this.binding.inCallActionRight.setImageResource(
1000 R.drawable.ic_bluetooth_audio_black_24dp);
1001 this.binding.inCallActionRight.setOnClickListener(null);
1002 this.binding.inCallActionRight.setClickable(false);
1003 break;
1004 }
1005 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1006 }
1007
1008 @SuppressLint("RestrictedApi")
1009 private void updateInCallButtonConfigurationVideo(
1010 final boolean videoEnabled, final boolean isCameraSwitchable) {
1011 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1012 if (isCameraSwitchable) {
1013 this.binding.inCallActionFarRight.setImageResource(
1014 R.drawable.ic_flip_camera_android_black_24dp);
1015 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1016 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1017 } else {
1018 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1019 }
1020 if (videoEnabled) {
1021 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1022 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1023 } else {
1024 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1025 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1026 }
1027 }
1028
1029 private void switchCamera(final View view) {
1030 Futures.addCallback(
1031 requireRtpConnection().switchCamera(),
1032 new FutureCallback<Boolean>() {
1033 @Override
1034 public void onSuccess(@NullableDecl Boolean isFrontCamera) {
1035 binding.localVideo.setMirror(isFrontCamera);
1036 }
1037
1038 @Override
1039 public void onFailure(@NonNull final Throwable throwable) {
1040 Log.d(
1041 Config.LOGTAG,
1042 "could not switch camera",
1043 Throwables.getRootCause(throwable));
1044 Toast.makeText(
1045 RtpSessionActivity.this,
1046 R.string.could_not_switch_camera,
1047 Toast.LENGTH_LONG)
1048 .show();
1049 }
1050 },
1051 MainThreadExecutor.getInstance());
1052 }
1053
1054 private void enableVideo(View view) {
1055 try {
1056 requireRtpConnection().setVideoEnabled(true);
1057 } catch (final IllegalStateException e) {
1058 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1059 return;
1060 }
1061 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1062 }
1063
1064 private void disableVideo(View view) {
1065 requireRtpConnection().setVideoEnabled(false);
1066 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1067 }
1068
1069 @SuppressLint("RestrictedApi")
1070 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1071 if (microphoneEnabled) {
1072 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1073 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1074 } else {
1075 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1076 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1077 }
1078 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1079 }
1080
1081 private void updateCallDuration() {
1082 final JingleRtpConnection connection =
1083 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1084 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1085 this.binding.duration.setVisibility(View.GONE);
1086 return;
1087 }
1088 if (connection.zeroDuration()) {
1089 this.binding.duration.setVisibility(View.GONE);
1090 } else {
1091 this.binding.duration.setText(
1092 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1093 this.binding.duration.setVisibility(View.VISIBLE);
1094 }
1095 }
1096
1097 private void updateVideoViews(final RtpEndUserState state) {
1098 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1099 binding.localVideo.setVisibility(View.GONE);
1100 binding.localVideo.release();
1101 binding.remoteVideoWrapper.setVisibility(View.GONE);
1102 binding.remoteVideo.release();
1103 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1104 if (isPictureInPicture()) {
1105 binding.appBarLayout.setVisibility(View.GONE);
1106 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1107 if (Arrays.asList(
1108 RtpEndUserState.APPLICATION_ERROR,
1109 RtpEndUserState.CONNECTIVITY_ERROR,
1110 RtpEndUserState.SECURITY_ERROR)
1111 .contains(state)) {
1112 binding.pipWarning.setVisibility(View.VISIBLE);
1113 binding.pipWaiting.setVisibility(View.GONE);
1114 } else {
1115 binding.pipWarning.setVisibility(View.GONE);
1116 binding.pipWaiting.setVisibility(View.GONE);
1117 }
1118 } else {
1119 binding.appBarLayout.setVisibility(View.VISIBLE);
1120 binding.pipPlaceholder.setVisibility(View.GONE);
1121 }
1122 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1123 return;
1124 }
1125 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1126 binding.localVideo.setVisibility(View.GONE);
1127 binding.remoteVideoWrapper.setVisibility(View.GONE);
1128 binding.appBarLayout.setVisibility(View.GONE);
1129 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1130 binding.pipWarning.setVisibility(View.GONE);
1131 binding.pipWaiting.setVisibility(View.VISIBLE);
1132 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1133 return;
1134 }
1135 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1136 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1137 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1138 // paint local view over remote view
1139 binding.localVideo.setZOrderMediaOverlay(true);
1140 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1141 addSink(localVideoTrack.get(), binding.localVideo);
1142 } else {
1143 binding.localVideo.setVisibility(View.GONE);
1144 }
1145 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1146 if (remoteVideoTrack.isPresent()) {
1147 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1148 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1149 binding.remoteVideo.setScalingType(
1150 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1151 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1152 if (state == RtpEndUserState.CONNECTED) {
1153 binding.appBarLayout.setVisibility(View.GONE);
1154 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1155 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1156 } else {
1157 binding.appBarLayout.setVisibility(View.VISIBLE);
1158 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1159 binding.remoteVideoWrapper.setVisibility(View.GONE);
1160 }
1161 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1162 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1163 } else {
1164 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1165 }
1166 } else {
1167 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1168 binding.remoteVideoWrapper.setVisibility(View.GONE);
1169 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1170 }
1171 }
1172
1173 private Optional<VideoTrack> getLocalVideoTrack() {
1174 final JingleRtpConnection connection =
1175 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1176 if (connection == null) {
1177 return Optional.absent();
1178 }
1179 return connection.getLocalVideoTrack();
1180 }
1181
1182 private Optional<VideoTrack> getRemoteVideoTrack() {
1183 final JingleRtpConnection connection =
1184 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1185 if (connection == null) {
1186 return Optional.absent();
1187 }
1188 return connection.getRemoteVideoTrack();
1189 }
1190
1191 private void disableMicrophone(View view) {
1192 final JingleRtpConnection rtpConnection = requireRtpConnection();
1193 if (rtpConnection.setMicrophoneEnabled(false)) {
1194 updateInCallButtonConfiguration();
1195 }
1196 }
1197
1198 private void enableMicrophone(View view) {
1199 final JingleRtpConnection rtpConnection = requireRtpConnection();
1200 if (rtpConnection.setMicrophoneEnabled(true)) {
1201 updateInCallButtonConfiguration();
1202 }
1203 }
1204
1205 private void switchToEarpiece(View view) {
1206 requireRtpConnection()
1207 .getAudioManager()
1208 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1209 acquireProximityWakeLock();
1210 }
1211
1212 private void switchToSpeaker(View view) {
1213 requireRtpConnection()
1214 .getAudioManager()
1215 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1216 releaseProximityWakeLock();
1217 }
1218
1219 private void retry(View view) {
1220 final Intent intent = getIntent();
1221 final Account account = extractAccount(intent);
1222 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1223 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1224 final String action = intent.getAction();
1225 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1226 this.rtpConnectionReference = null;
1227 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1228 proposeJingleRtpSession(account, with, media);
1229 }
1230
1231 private void exit(final View view) {
1232 finish();
1233 }
1234
1235 private void recordVoiceMail(final View view) {
1236 final Intent intent = getIntent();
1237 final Account account = extractAccount(intent);
1238 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1239 final Conversation conversation =
1240 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1241 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1242 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1243 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1244 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1245 launchIntent.putExtra(
1246 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1247 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1248 startActivity(launchIntent);
1249 finish();
1250 }
1251
1252 private Contact getWith() {
1253 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1254 final Account account = id.account;
1255 return account.getRoster().getContact(id.with);
1256 }
1257
1258 private JingleRtpConnection requireRtpConnection() {
1259 final JingleRtpConnection connection =
1260 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1261 if (connection == null) {
1262 throw new IllegalStateException("No RTP connection found");
1263 }
1264 return connection;
1265 }
1266
1267 @Override
1268 public void onJingleRtpConnectionUpdate(
1269 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1270 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1271 if (END_CARD.contains(state)) {
1272 Log.d(Config.LOGTAG, "end card reached");
1273 releaseProximityWakeLock();
1274 runOnUiThread(
1275 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1276 }
1277 if (with.isBareJid()) {
1278 updateRtpSessionProposalState(account, with, state);
1279 return;
1280 }
1281 if (emptyReference(this.rtpConnectionReference)) {
1282 if (END_CARD.contains(state)) {
1283 Log.d(Config.LOGTAG, "not reinitializing session");
1284 return;
1285 }
1286 // this happens when going from proposed session to actual session
1287 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1288 return;
1289 }
1290 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1291 final boolean verified = requireRtpConnection().isVerified();
1292 final Set<Media> media = getMedia();
1293 final Contact contact = getWith();
1294 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1295 if (state == RtpEndUserState.ENDED) {
1296 finish();
1297 return;
1298 }
1299 runOnUiThread(
1300 () -> {
1301 updateStateDisplay(state, media);
1302 updateVerifiedShield(
1303 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1304 updateButtonConfiguration(state, media);
1305 updateVideoViews(state);
1306 updateIncomingCallScreen(state, contact);
1307 invalidateOptionsMenu();
1308 });
1309 if (END_CARD.contains(state)) {
1310 final JingleRtpConnection rtpConnection = requireRtpConnection();
1311 resetIntent(account, with, state, rtpConnection.getMedia());
1312 releaseVideoTracks(rtpConnection);
1313 this.rtpConnectionReference = null;
1314 }
1315 } else {
1316 Log.d(Config.LOGTAG, "received update for other rtp session");
1317 }
1318 }
1319
1320 @Override
1321 public void onAudioDeviceChanged(
1322 AppRTCAudioManager.AudioDevice selectedAudioDevice,
1323 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1324 Log.d(
1325 Config.LOGTAG,
1326 "onAudioDeviceChanged in activity: selected:"
1327 + selectedAudioDevice
1328 + ", available:"
1329 + availableAudioDevices);
1330 try {
1331 if (getMedia().contains(Media.VIDEO)) {
1332 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1333 return;
1334 }
1335 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1336 if (endUserState == RtpEndUserState.CONNECTED) {
1337 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1338 updateInCallButtonConfigurationSpeaker(
1339 audioManager.getSelectedAudioDevice(),
1340 audioManager.getAudioDevices().size());
1341 } else if (END_CARD.contains(endUserState)) {
1342 Log.d(
1343 Config.LOGTAG,
1344 "onAudioDeviceChanged() nothing to do because end card has been reached");
1345 } else {
1346 putProximityWakeLockInProperState(selectedAudioDevice);
1347 }
1348 } catch (IllegalStateException e) {
1349 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1350 }
1351 }
1352
1353 @Override
1354 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1355 super.onSaveInstanceState(outState);
1356 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1357 }
1358
1359 private void updateRtpSessionProposalState(
1360 final Account account, final Jid with, final RtpEndUserState state) {
1361 final Intent currentIntent = getIntent();
1362 final String withExtra =
1363 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1364 if (withExtra == null) {
1365 return;
1366 }
1367 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1368 runOnUiThread(
1369 () -> {
1370 updateVerifiedShield(false);
1371 updateStateDisplay(state);
1372 updateButtonConfiguration(state);
1373 updateIncomingCallScreen(state);
1374 invalidateOptionsMenu();
1375 });
1376 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1377 }
1378 }
1379
1380 private void resetIntent(final Bundle extras) {
1381 final Intent intent = new Intent(Intent.ACTION_VIEW);
1382 intent.putExtras(extras);
1383 setIntent(intent);
1384 }
1385
1386 private void resetIntent(
1387 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1388 final Intent intent = new Intent(Intent.ACTION_VIEW);
1389 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1390 if (account.getRoster()
1391 .getContact(with)
1392 .getPresences()
1393 .anySupport(Namespace.JINGLE_MESSAGE)) {
1394 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1395 } else {
1396 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1397 }
1398 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1399 intent.putExtra(
1400 EXTRA_LAST_ACTION,
1401 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1402 setIntent(intent);
1403 }
1404
1405 private static boolean emptyReference(final WeakReference<?> weakReference) {
1406 return weakReference == null || weakReference.get() == null;
1407 }
1408}