1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.PictureInPictureParams;
9import android.content.ActivityNotFoundException;
10import android.content.Context;
11import android.content.Intent;
12import android.content.pm.PackageManager;
13import android.os.Build;
14import android.os.Bundle;
15import android.os.Handler;
16import android.os.PowerManager;
17import android.os.SystemClock;
18import android.util.Log;
19import android.util.Rational;
20import android.view.KeyEvent;
21import android.view.Menu;
22import android.view.MenuItem;
23import android.view.View;
24import android.view.WindowManager;
25import android.widget.Toast;
26
27import androidx.annotation.NonNull;
28import androidx.annotation.Nullable;
29import androidx.annotation.RequiresApi;
30import androidx.annotation.StringRes;
31import androidx.databinding.DataBindingUtil;
32
33import com.google.common.base.Optional;
34import com.google.common.base.Preconditions;
35import com.google.common.base.Throwables;
36import com.google.common.collect.ImmutableList;
37import com.google.common.collect.ImmutableSet;
38import com.google.common.util.concurrent.FutureCallback;
39import com.google.common.util.concurrent.Futures;
40
41import org.jetbrains.annotations.NotNull;
42import org.webrtc.RendererCommon;
43import org.webrtc.SurfaceViewRenderer;
44import org.webrtc.VideoTrack;
45
46import java.lang.ref.WeakReference;
47import java.util.Arrays;
48import java.util.Collections;
49import java.util.List;
50import java.util.Set;
51
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.R;
54import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.services.AppRTCAudioManager;
59import eu.siacs.conversations.services.XmppConnectionService;
60import eu.siacs.conversations.ui.widget.DialpadView;
61import eu.siacs.conversations.ui.util.AvatarWorkerTask;
62import eu.siacs.conversations.ui.util.MainThreadExecutor;
63import eu.siacs.conversations.ui.util.Rationals;
64import eu.siacs.conversations.utils.PermissionUtils;
65import eu.siacs.conversations.utils.TimeFrameUtils;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
69import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
70import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
71import eu.siacs.conversations.xmpp.jingle.Media;
72import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
73
74public class RtpSessionActivity extends XmppActivity
75 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
76 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
77
78 public static final String EXTRA_WITH = "with";
79 public static final String EXTRA_SESSION_ID = "session_id";
80 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
81 public static final String EXTRA_LAST_ACTION = "last_action";
82 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
83 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
84 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
85
86 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
87
88 public static final List<RtpEndUserState> END_CARD =
89 Arrays.asList(
90 RtpEndUserState.APPLICATION_ERROR,
91 RtpEndUserState.SECURITY_ERROR,
92 RtpEndUserState.DECLINED_OR_BUSY,
93 RtpEndUserState.CONNECTIVITY_ERROR,
94 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
95 RtpEndUserState.RETRACTED);
96 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
97 Arrays.asList(
98 RtpEndUserState.APPLICATION_ERROR,
99 RtpEndUserState.CONNECTIVITY_ERROR,
100 RtpEndUserState.SECURITY_ERROR);
101 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
102 Arrays.asList(
103 RtpEndUserState.CONNECTING,
104 RtpEndUserState.CONNECTED,
105 RtpEndUserState.RECONNECTING);
106 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
107 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
108 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
109 Arrays.asList(
110 RtpEndUserState.ACCEPTING_CALL,
111 RtpEndUserState.CONNECTING,
112 RtpEndUserState.RECONNECTING);
113 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
114 private static final int REQUEST_ACCEPT_CALL = 0x1111;
115 private WeakReference<JingleRtpConnection> rtpConnectionReference;
116
117 private ActivityRtpSessionBinding binding;
118 private PowerManager.WakeLock mProximityWakeLock;
119
120 private final Handler mHandler = new Handler();
121 private final Runnable mTickExecutor =
122 new Runnable() {
123 @Override
124 public void run() {
125 updateCallDuration();
126 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
127 }
128 };
129
130 private static Set<Media> actionToMedia(final String action) {
131 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
132 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
133 } else {
134 return ImmutableSet.of(Media.AUDIO);
135 }
136 }
137
138 private static void addSink(
139 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
140 try {
141 videoTrack.addSink(surfaceViewRenderer);
142 } catch (final IllegalStateException e) {
143 Log.e(
144 Config.LOGTAG,
145 "possible race condition on trying to display video track. ignoring",
146 e);
147 }
148 }
149
150 @Override
151 public void onCreate(Bundle savedInstanceState) {
152 super.onCreate(savedInstanceState);
153 getWindow()
154 .addFlags(
155 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
156 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
157 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
158 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
159 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
160 setSupportActionBar(binding.toolbar);
161
162 binding.dialpad.setClickConsumer(tag -> {
163 final JingleRtpConnection connection =
164 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
165 if (connection != null) connection.applyDtmfTone(tag);
166 });
167
168 if (savedInstanceState != null) {
169 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
170 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
171 }
172 }
173
174 @Override
175 public boolean onCreateOptionsMenu(final Menu menu) {
176 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
177 final MenuItem help = menu.findItem(R.id.action_help);
178 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
179 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
180 help.setVisible(isHelpButtonVisible());
181 gotoChat.setVisible(isSwitchToConversationVisible());
182 dialpad.setVisible(isAudioOnlyConversation());
183 return super.onCreateOptionsMenu(menu);
184 }
185
186 @Override
187 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
188 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
189 if (xmppConnectionService != null) {
190 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
191 return true;
192 }
193 }
194 }
195 return super.onKeyDown(keyCode, event);
196 }
197
198 private boolean isHelpButtonVisible() {
199 try {
200 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
201 } catch (IllegalStateException e) {
202 final Intent intent = getIntent();
203 final String state =
204 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
205 if (state != null) {
206 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
207 } else {
208 return false;
209 }
210 }
211 }
212
213 private boolean isSwitchToConversationVisible() {
214 final JingleRtpConnection connection =
215 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
216 return connection != null
217 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
218 }
219
220 private boolean isAudioOnlyConversation() {
221 final JingleRtpConnection connection =
222 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
223
224 return connection != null && !connection.getMedia().contains(Media.VIDEO);
225 }
226
227 private void switchToConversation() {
228 final Contact contact = getWith();
229 final Conversation conversation =
230 xmppConnectionService.findOrCreateConversation(
231 contact.getAccount(), contact.getJid(), false, true);
232 switchToConversation(conversation);
233 }
234
235 private void toggleDialpadVisibility() {
236 if (binding.dialpad.getVisibility() == View.VISIBLE) {
237 binding.dialpad.setVisibility(View.GONE);
238 }
239 else {
240 binding.dialpad.setVisibility(View.VISIBLE);
241 }
242 }
243
244 public boolean onOptionsItemSelected(final MenuItem item) {
245 switch (item.getItemId()) {
246 case R.id.action_help:
247 launchHelpInBrowser();
248 break;
249 case R.id.action_goto_chat:
250 switchToConversation();
251 break;
252 case R.id.action_dialpad:
253 toggleDialpadVisibility();
254 break;
255 }
256 return super.onOptionsItemSelected(item);
257 }
258
259 private void launchHelpInBrowser() {
260 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
261 try {
262 startActivity(intent);
263 } catch (final ActivityNotFoundException e) {
264 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
265 .show();
266 }
267 }
268
269 private void endCall(View view) {
270 endCall();
271 }
272
273 private void endCall() {
274 if (this.rtpConnectionReference == null) {
275 retractSessionProposal();
276 finish();
277 } else {
278 requireRtpConnection().endCall();
279 }
280 }
281
282 private void retractSessionProposal() {
283 final Intent intent = getIntent();
284 final String action = intent.getAction();
285 final Account account = extractAccount(intent);
286 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
287 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
288 if (!Intent.ACTION_VIEW.equals(action)
289 || state == null
290 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
291 resetIntent(
292 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
293 }
294 xmppConnectionService
295 .getJingleConnectionManager()
296 .retractSessionProposal(account, with.asBareJid());
297 }
298
299 private void rejectCall(View view) {
300 requireRtpConnection().rejectCall();
301 finish();
302 }
303
304 private void acceptCall(View view) {
305 requestPermissionsAndAcceptCall();
306 }
307
308 private void requestPermissionsAndAcceptCall() {
309 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
310 if (getMedia().contains(Media.VIDEO)) {
311 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
312 } else {
313 permissions.add(Manifest.permission.RECORD_AUDIO);
314 }
315 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
316 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
317 }
318 if (PermissionUtils.hasPermission(this, permissions.build(), REQUEST_ACCEPT_CALL)) {
319 putScreenInCallMode();
320 checkRecorderAndAcceptCall();
321 }
322 }
323
324 private void checkRecorderAndAcceptCall() {
325 checkMicrophoneAvailabilityAsync();
326 try {
327 requireRtpConnection().acceptCall();
328 } catch (final IllegalStateException e) {
329 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
330 }
331 }
332
333 private void checkMicrophoneAvailabilityAsync() {
334 new Thread(this::checkMicrophoneAvailability).start();
335 }
336
337 private void checkMicrophoneAvailability() {
338 final long start = SystemClock.elapsedRealtime();
339 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
340 final long stop = SystemClock.elapsedRealtime();
341 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
342 if (isMicrophoneAvailable) {
343 return;
344 }
345 runOnUiThread(
346 () ->
347 Toast.makeText(this, R.string.microphone_unavailable, Toast.LENGTH_LONG)
348 .show());
349 }
350
351 private void putScreenInCallMode() {
352 putScreenInCallMode(requireRtpConnection().getMedia());
353 }
354
355 private void putScreenInCallMode(final Set<Media> media) {
356 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
357 if (!media.contains(Media.VIDEO)) {
358 final JingleRtpConnection rtpConnection =
359 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
360 final AppRTCAudioManager audioManager =
361 rtpConnection == null ? null : rtpConnection.getAudioManager();
362 if (audioManager == null
363 || audioManager.getSelectedAudioDevice()
364 == AppRTCAudioManager.AudioDevice.EARPIECE) {
365 acquireProximityWakeLock();
366 }
367 }
368 }
369
370 @SuppressLint("WakelockTimeout")
371 private void acquireProximityWakeLock() {
372 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
373 if (powerManager == null) {
374 Log.e(Config.LOGTAG, "power manager not available");
375 return;
376 }
377 if (isFinishing()) {
378 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
379 return;
380 }
381 if (this.mProximityWakeLock == null) {
382 this.mProximityWakeLock =
383 powerManager.newWakeLock(
384 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
385 }
386 if (!this.mProximityWakeLock.isHeld()) {
387 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
388 this.mProximityWakeLock.acquire();
389 }
390 }
391
392 private void releaseProximityWakeLock() {
393 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
394 Log.d(Config.LOGTAG, "releasing proximity wake lock");
395 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
396 this.mProximityWakeLock = null;
397 }
398 }
399
400 private void putProximityWakeLockInProperState(
401 final AppRTCAudioManager.AudioDevice audioDevice) {
402 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
403 acquireProximityWakeLock();
404 } else {
405 releaseProximityWakeLock();
406 }
407 }
408
409 @Override
410 protected void refreshUiReal() {}
411
412 @Override
413 public void onNewIntent(final Intent intent) {
414 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
415 super.onNewIntent(intent);
416 setIntent(intent);
417 if (xmppConnectionService == null) {
418 Log.d(
419 Config.LOGTAG,
420 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
421 return;
422 }
423 final Account account = extractAccount(intent);
424 final String action = intent.getAction();
425 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
426 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
427 if (sessionId != null) {
428 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
429 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
430 return;
431 }
432 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
433 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
434 requestPermissionsAndAcceptCall();
435 resetIntent(intent.getExtras());
436 }
437 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
438 proposeJingleRtpSession(account, with, actionToMedia(action));
439 setWith(account.getRoster().getContact(with), null);
440 } else {
441 throw new IllegalStateException("received onNewIntent without sessionId");
442 }
443 }
444
445 @Override
446 void onBackendConnected() {
447 final Intent intent = getIntent();
448 final String action = intent.getAction();
449 final Account account = extractAccount(intent);
450 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
451 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
452 if (sessionId != null) {
453 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
454 return;
455 }
456 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
457 Log.d(Config.LOGTAG, "intent action was accept");
458 requestPermissionsAndAcceptCall();
459 resetIntent(intent.getExtras());
460 }
461 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
462 proposeJingleRtpSession(account, with, actionToMedia(action));
463 setWith(account.getRoster().getContact(with), null);
464 } else if (Intent.ACTION_VIEW.equals(action)) {
465 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
466 final RtpEndUserState state =
467 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
468 if (state != null) {
469 Log.d(Config.LOGTAG, "restored last state from intent extra");
470 updateButtonConfiguration(state);
471 updateVerifiedShield(false);
472 updateStateDisplay(state);
473 updateIncomingCallScreen(state);
474 invalidateOptionsMenu();
475 }
476 setWith(account.getRoster().getContact(with), state);
477 if (xmppConnectionService
478 .getJingleConnectionManager()
479 .fireJingleRtpConnectionStateUpdates()) {
480 return;
481 }
482 if (END_CARD.contains(state)
483 || xmppConnectionService
484 .getJingleConnectionManager()
485 .hasMatchingProposal(account, with)) {
486 return;
487 }
488 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
489 finish();
490 }
491 }
492
493 private void setWith(final RtpEndUserState state) {
494 setWith(getWith(), state);
495 }
496
497 private void setWith(final Contact contact, final RtpEndUserState state) {
498 binding.with.setText(contact.getDisplayName());
499 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
500 .contains(state)) {
501 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
502 binding.withJid.setVisibility(View.VISIBLE);
503 } else {
504 binding.withJid.setVisibility(View.GONE);
505 }
506 }
507
508 private void proposeJingleRtpSession(
509 final Account account, final Jid with, final Set<Media> media) {
510 checkMicrophoneAvailabilityAsync();
511 if (with.isBareJid()) {
512 xmppConnectionService
513 .getJingleConnectionManager()
514 .proposeJingleRtpSession(account, with, media);
515 } else {
516 final String sessionId =
517 xmppConnectionService
518 .getJingleConnectionManager()
519 .initializeRtpSession(account, with, media);
520 initializeActivityWithRunningRtpSession(account, with, sessionId);
521 resetIntent(account, with, sessionId);
522 }
523 putScreenInCallMode(media);
524 }
525
526 @Override
527 public void onRequestPermissionsResult(
528 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
529 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
530 final PermissionUtils.PermissionResult permissionResult =
531 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
532 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
533 if (requestCode == REQUEST_ACCEPT_CALL) {
534 checkRecorderAndAcceptCall();
535 }
536 } else {
537 @StringRes int res;
538 final String firstDenied =
539 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
540 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
541 res = R.string.no_microphone_permission;
542 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
543 res = R.string.no_camera_permission;
544 } else {
545 throw new IllegalStateException("Invalid permission result request");
546 }
547 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
548 .show();
549 }
550 }
551
552 @Override
553 public void onStart() {
554 super.onStart();
555 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
556 this.binding.remoteVideo.setOnAspectRatioChanged(this);
557 }
558
559 @Override
560 public void onStop() {
561 mHandler.removeCallbacks(mTickExecutor);
562 binding.remoteVideo.release();
563 binding.remoteVideo.setOnAspectRatioChanged(null);
564 binding.localVideo.release();
565 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
566 final JingleRtpConnection jingleRtpConnection =
567 weakReference == null ? null : weakReference.get();
568 if (jingleRtpConnection != null) {
569 releaseVideoTracks(jingleRtpConnection);
570 }
571 releaseProximityWakeLock();
572 super.onStop();
573 }
574
575 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
576 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
577 if (remoteVideo.isPresent()) {
578 remoteVideo.get().removeSink(binding.remoteVideo);
579 }
580 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
581 if (localVideo.isPresent()) {
582 localVideo.get().removeSink(binding.localVideo);
583 }
584 }
585
586 @Override
587 public void onBackPressed() {
588 if (isConnected()) {
589 if (switchToPictureInPicture()) {
590 return;
591 }
592 } else {
593 endCall();
594 }
595 super.onBackPressed();
596 }
597
598 @Override
599 public void onUserLeaveHint() {
600 super.onUserLeaveHint();
601 if (switchToPictureInPicture()) {
602 return;
603 }
604 // TODO apparently this method is not getting called on Android 10 when using the task
605 // switcher
606 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
607 retractSessionProposal();
608 }
609 }
610
611 private boolean isConnected() {
612 final JingleRtpConnection connection =
613 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
614 return connection != null
615 && STATES_CONSIDERED_CONNECTED.contains(connection.getEndUserState());
616 }
617
618 private boolean switchToPictureInPicture() {
619 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
620 if (shouldBePictureInPicture()) {
621 startPictureInPicture();
622 return true;
623 }
624 }
625 return false;
626 }
627
628 @RequiresApi(api = Build.VERSION_CODES.O)
629 private void startPictureInPicture() {
630 try {
631 final Rational rational = this.binding.remoteVideo.getAspectRatio();
632 final Rational clippedRational = Rationals.clip(rational);
633 Log.d(
634 Config.LOGTAG,
635 "suggested rational " + rational + ". clipped to " + clippedRational);
636 enterPictureInPictureMode(
637 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
638 } catch (final IllegalStateException e) {
639 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
640 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
641 }
642 }
643
644 @Override
645 public void onAspectRatioChanged(final Rational rational) {
646 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
647 final Rational clippedRational = Rationals.clip(rational);
648 Log.d(
649 Config.LOGTAG,
650 "suggested rational after aspect ratio change "
651 + rational
652 + ". clipped to "
653 + clippedRational);
654 setPictureInPictureParams(
655 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
656 }
657 }
658
659 private boolean deviceSupportsPictureInPicture() {
660 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
661 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
662 } else {
663 return false;
664 }
665 }
666
667 private boolean shouldBePictureInPicture() {
668 try {
669 final JingleRtpConnection rtpConnection = requireRtpConnection();
670 return rtpConnection.getMedia().contains(Media.VIDEO)
671 && Arrays.asList(
672 RtpEndUserState.ACCEPTING_CALL,
673 RtpEndUserState.CONNECTING,
674 RtpEndUserState.CONNECTED)
675 .contains(rtpConnection.getEndUserState());
676 } catch (final IllegalStateException e) {
677 return false;
678 }
679 }
680
681 private boolean initializeActivityWithRunningRtpSession(
682 final Account account, Jid with, String sessionId) {
683 final WeakReference<JingleRtpConnection> reference =
684 xmppConnectionService
685 .getJingleConnectionManager()
686 .findJingleRtpConnection(account, with, sessionId);
687 if (reference == null || reference.get() == null) {
688 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
689 xmppConnectionService
690 .getJingleConnectionManager()
691 .getTerminalSessionState(with, sessionId);
692 if (terminatedRtpSession == null) {
693 throw new IllegalStateException(
694 "failed to initialize activity with running rtp session. session not found");
695 }
696 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
697 return true;
698 }
699 this.rtpConnectionReference = reference;
700 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
701 final boolean verified = requireRtpConnection().isVerified();
702 if (currentState == RtpEndUserState.ENDED) {
703 finish();
704 return true;
705 }
706 final Set<Media> media = getMedia();
707 if (currentState == RtpEndUserState.INCOMING_CALL) {
708 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
709 }
710 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
711 requireRtpConnection().getState())) {
712 putScreenInCallMode();
713 }
714 setWith(currentState);
715 updateVideoViews(currentState);
716 updateStateDisplay(currentState, media);
717 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
718 updateButtonConfiguration(currentState, media);
719 updateIncomingCallScreen(currentState);
720 invalidateOptionsMenu();
721 return false;
722 }
723
724 private void initializeWithTerminatedSessionState(
725 final Account account,
726 final Jid with,
727 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
728 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
729 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
730 finish();
731 return;
732 }
733 final RtpEndUserState state = terminatedRtpSession.state;
734 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
735 updateButtonConfiguration(state);
736 updateStateDisplay(state);
737 updateIncomingCallScreen(state);
738 updateCallDuration();
739 updateVerifiedShield(false);
740 invalidateOptionsMenu();
741 setWith(account.getRoster().getContact(with), state);
742 }
743
744 private void reInitializeActivityWithRunningRtpSession(
745 final Account account, Jid with, String sessionId) {
746 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
747 resetIntent(account, with, sessionId);
748 }
749
750 private void resetIntent(final Account account, final Jid with, final String sessionId) {
751 final Intent intent = new Intent(Intent.ACTION_VIEW);
752 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
753 intent.putExtra(EXTRA_WITH, with.toEscapedString());
754 intent.putExtra(EXTRA_SESSION_ID, sessionId);
755 setIntent(intent);
756 }
757
758 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
759 surfaceViewRenderer.setVisibility(View.VISIBLE);
760 try {
761 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
762 } catch (final IllegalStateException e) {
763 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
764 }
765 surfaceViewRenderer.setEnableHardwareScaler(true);
766 }
767
768 private void updateStateDisplay(final RtpEndUserState state) {
769 updateStateDisplay(state, Collections.emptySet());
770 }
771
772 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media) {
773 switch (state) {
774 case INCOMING_CALL:
775 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
776 if (media.contains(Media.VIDEO)) {
777 setTitle(R.string.rtp_state_incoming_video_call);
778 } else {
779 setTitle(R.string.rtp_state_incoming_call);
780 }
781 break;
782 case CONNECTING:
783 setTitle(R.string.rtp_state_connecting);
784 break;
785 case CONNECTED:
786 setTitle(R.string.rtp_state_connected);
787 break;
788 case RECONNECTING:
789 setTitle(R.string.rtp_state_reconnecting);
790 break;
791 case ACCEPTING_CALL:
792 setTitle(R.string.rtp_state_accepting_call);
793 break;
794 case ENDING_CALL:
795 setTitle(R.string.rtp_state_ending_call);
796 break;
797 case FINDING_DEVICE:
798 setTitle(R.string.rtp_state_finding_device);
799 break;
800 case RINGING:
801 setTitle(R.string.rtp_state_ringing);
802 break;
803 case DECLINED_OR_BUSY:
804 setTitle(R.string.rtp_state_declined_or_busy);
805 break;
806 case CONNECTIVITY_ERROR:
807 setTitle(R.string.rtp_state_connectivity_error);
808 break;
809 case CONNECTIVITY_LOST_ERROR:
810 setTitle(R.string.rtp_state_connectivity_lost_error);
811 break;
812 case RETRACTED:
813 setTitle(R.string.rtp_state_retracted);
814 break;
815 case APPLICATION_ERROR:
816 setTitle(R.string.rtp_state_application_failure);
817 break;
818 case SECURITY_ERROR:
819 setTitle(R.string.rtp_state_security_error);
820 break;
821 case ENDED:
822 throw new IllegalStateException(
823 "Activity should have called finishAndReleaseWakeLock();");
824 default:
825 throw new IllegalStateException(
826 String.format("State %s has not been handled in UI", state));
827 }
828 }
829
830 private void updateVerifiedShield(final boolean verified) {
831 if (isPictureInPicture()) {
832 this.binding.verified.setVisibility(View.GONE);
833 return;
834 }
835 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
836 }
837
838 private void updateIncomingCallScreen(final RtpEndUserState state) {
839 updateIncomingCallScreen(state, null);
840 }
841
842 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
843 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
844 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
845 if (show) {
846 binding.contactPhoto.setVisibility(View.VISIBLE);
847 if (contact == null) {
848 AvatarWorkerTask.loadAvatar(
849 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
850 } else {
851 AvatarWorkerTask.loadAvatar(
852 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
853 }
854 } else {
855 binding.contactPhoto.setVisibility(View.GONE);
856 }
857 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
858 binding.usingAccount.setVisibility(View.VISIBLE);
859 binding.usingAccount.setText(
860 getString(
861 R.string.using_account,
862 account.getJid().asBareJid().toEscapedString()));
863 } else {
864 binding.usingAccount.setVisibility(View.GONE);
865 binding.contactPhoto.setVisibility(View.GONE);
866 }
867 }
868
869 private Set<Media> getMedia() {
870 return requireRtpConnection().getMedia();
871 }
872
873 private void updateButtonConfiguration(final RtpEndUserState state) {
874 updateButtonConfiguration(state, Collections.emptySet());
875 }
876
877 @SuppressLint("RestrictedApi")
878 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
879 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
880 this.binding.rejectCall.setVisibility(View.INVISIBLE);
881 this.binding.endCall.setVisibility(View.INVISIBLE);
882 this.binding.acceptCall.setVisibility(View.INVISIBLE);
883 } else if (state == RtpEndUserState.INCOMING_CALL) {
884 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
885 this.binding.rejectCall.setOnClickListener(this::rejectCall);
886 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
887 this.binding.rejectCall.setVisibility(View.VISIBLE);
888 this.binding.endCall.setVisibility(View.INVISIBLE);
889 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
890 this.binding.acceptCall.setOnClickListener(this::acceptCall);
891 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
892 this.binding.acceptCall.setVisibility(View.VISIBLE);
893 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
894 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
895 this.binding.rejectCall.setOnClickListener(this::exit);
896 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
897 this.binding.rejectCall.setVisibility(View.VISIBLE);
898 this.binding.endCall.setVisibility(View.INVISIBLE);
899 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
900 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
901 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
902 this.binding.acceptCall.setVisibility(View.VISIBLE);
903 } else if (asList(
904 RtpEndUserState.CONNECTIVITY_ERROR,
905 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
906 RtpEndUserState.APPLICATION_ERROR,
907 RtpEndUserState.RETRACTED,
908 RtpEndUserState.SECURITY_ERROR)
909 .contains(state)) {
910 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
911 this.binding.rejectCall.setOnClickListener(this::exit);
912 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
913 this.binding.rejectCall.setVisibility(View.VISIBLE);
914 this.binding.endCall.setVisibility(View.INVISIBLE);
915 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
916 this.binding.acceptCall.setOnClickListener(this::retry);
917 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
918 this.binding.acceptCall.setVisibility(View.VISIBLE);
919 } else {
920 this.binding.rejectCall.setVisibility(View.INVISIBLE);
921 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
922 this.binding.endCall.setOnClickListener(this::endCall);
923 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
924 this.binding.endCall.setVisibility(View.VISIBLE);
925 this.binding.acceptCall.setVisibility(View.INVISIBLE);
926 }
927 updateInCallButtonConfiguration(state, media);
928 }
929
930 private boolean isPictureInPicture() {
931 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
932 return isInPictureInPictureMode();
933 } else {
934 return false;
935 }
936 }
937
938 private void updateInCallButtonConfiguration() {
939 updateInCallButtonConfiguration(
940 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
941 }
942
943 @SuppressLint("RestrictedApi")
944 private void updateInCallButtonConfiguration(
945 final RtpEndUserState state, final Set<Media> media) {
946 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
947 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
948 if (media.contains(Media.VIDEO)) {
949 final JingleRtpConnection rtpConnection = requireRtpConnection();
950 updateInCallButtonConfigurationVideo(
951 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
952 } else {
953 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
954 updateInCallButtonConfigurationSpeaker(
955 audioManager.getSelectedAudioDevice(),
956 audioManager.getAudioDevices().size());
957 this.binding.inCallActionFarRight.setVisibility(View.GONE);
958 }
959 if (media.contains(Media.AUDIO)) {
960 updateInCallButtonConfigurationMicrophone(
961 requireRtpConnection().isMicrophoneEnabled());
962 } else {
963 this.binding.inCallActionLeft.setVisibility(View.GONE);
964 }
965 } else {
966 this.binding.inCallActionLeft.setVisibility(View.GONE);
967 this.binding.inCallActionRight.setVisibility(View.GONE);
968 this.binding.inCallActionFarRight.setVisibility(View.GONE);
969 }
970 }
971
972 @SuppressLint("RestrictedApi")
973 private void updateInCallButtonConfigurationSpeaker(
974 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
975 switch (selectedAudioDevice) {
976 case EARPIECE:
977 this.binding.inCallActionRight.setImageResource(
978 R.drawable.ic_volume_off_black_24dp);
979 if (numberOfChoices >= 2) {
980 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
981 } else {
982 this.binding.inCallActionRight.setOnClickListener(null);
983 this.binding.inCallActionRight.setClickable(false);
984 }
985 break;
986 case WIRED_HEADSET:
987 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
988 this.binding.inCallActionRight.setOnClickListener(null);
989 this.binding.inCallActionRight.setClickable(false);
990 break;
991 case SPEAKER_PHONE:
992 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
993 if (numberOfChoices >= 2) {
994 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
995 } else {
996 this.binding.inCallActionRight.setOnClickListener(null);
997 this.binding.inCallActionRight.setClickable(false);
998 }
999 break;
1000 case BLUETOOTH:
1001 this.binding.inCallActionRight.setImageResource(
1002 R.drawable.ic_bluetooth_audio_black_24dp);
1003 this.binding.inCallActionRight.setOnClickListener(null);
1004 this.binding.inCallActionRight.setClickable(false);
1005 break;
1006 }
1007 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1008 }
1009
1010 @SuppressLint("RestrictedApi")
1011 private void updateInCallButtonConfigurationVideo(
1012 final boolean videoEnabled, final boolean isCameraSwitchable) {
1013 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1014 if (isCameraSwitchable) {
1015 this.binding.inCallActionFarRight.setImageResource(
1016 R.drawable.ic_flip_camera_android_black_24dp);
1017 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1018 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1019 } else {
1020 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1021 }
1022 if (videoEnabled) {
1023 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1024 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1025 } else {
1026 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1027 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1028 }
1029 }
1030
1031 private void switchCamera(final View view) {
1032 Futures.addCallback(
1033 requireRtpConnection().switchCamera(),
1034 new FutureCallback<Boolean>() {
1035 @Override
1036 public void onSuccess(@Nullable Boolean isFrontCamera) {
1037 binding.localVideo.setMirror(isFrontCamera);
1038 }
1039
1040 @Override
1041 public void onFailure(@NonNull final Throwable throwable) {
1042 Log.d(
1043 Config.LOGTAG,
1044 "could not switch camera",
1045 Throwables.getRootCause(throwable));
1046 Toast.makeText(
1047 RtpSessionActivity.this,
1048 R.string.could_not_switch_camera,
1049 Toast.LENGTH_LONG)
1050 .show();
1051 }
1052 },
1053 MainThreadExecutor.getInstance());
1054 }
1055
1056 private void enableVideo(View view) {
1057 try {
1058 requireRtpConnection().setVideoEnabled(true);
1059 } catch (final IllegalStateException e) {
1060 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1061 return;
1062 }
1063 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1064 }
1065
1066 private void disableVideo(View view) {
1067 requireRtpConnection().setVideoEnabled(false);
1068 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1069 }
1070
1071 @SuppressLint("RestrictedApi")
1072 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1073 if (microphoneEnabled) {
1074 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1075 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1076 } else {
1077 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1078 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1079 }
1080 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1081 }
1082
1083 private void updateCallDuration() {
1084 final JingleRtpConnection connection =
1085 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1086 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1087 this.binding.duration.setVisibility(View.GONE);
1088 return;
1089 }
1090 if (connection.zeroDuration()) {
1091 this.binding.duration.setVisibility(View.GONE);
1092 } else {
1093 this.binding.duration.setText(
1094 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1095 this.binding.duration.setVisibility(View.VISIBLE);
1096 }
1097 }
1098
1099 private void updateVideoViews(final RtpEndUserState state) {
1100 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1101 binding.localVideo.setVisibility(View.GONE);
1102 binding.localVideo.release();
1103 binding.remoteVideoWrapper.setVisibility(View.GONE);
1104 binding.remoteVideo.release();
1105 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1106 if (isPictureInPicture()) {
1107 binding.appBarLayout.setVisibility(View.GONE);
1108 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1109 if (Arrays.asList(
1110 RtpEndUserState.APPLICATION_ERROR,
1111 RtpEndUserState.CONNECTIVITY_ERROR,
1112 RtpEndUserState.SECURITY_ERROR)
1113 .contains(state)) {
1114 binding.pipWarning.setVisibility(View.VISIBLE);
1115 binding.pipWaiting.setVisibility(View.GONE);
1116 } else {
1117 binding.pipWarning.setVisibility(View.GONE);
1118 binding.pipWaiting.setVisibility(View.GONE);
1119 }
1120 } else {
1121 binding.appBarLayout.setVisibility(View.VISIBLE);
1122 binding.pipPlaceholder.setVisibility(View.GONE);
1123 }
1124 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1125 return;
1126 }
1127 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1128 binding.localVideo.setVisibility(View.GONE);
1129 binding.remoteVideoWrapper.setVisibility(View.GONE);
1130 binding.appBarLayout.setVisibility(View.GONE);
1131 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1132 binding.pipWarning.setVisibility(View.GONE);
1133 binding.pipWaiting.setVisibility(View.VISIBLE);
1134 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1135 return;
1136 }
1137 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1138 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1139 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1140 // paint local view over remote view
1141 binding.localVideo.setZOrderMediaOverlay(true);
1142 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1143 addSink(localVideoTrack.get(), binding.localVideo);
1144 } else {
1145 binding.localVideo.setVisibility(View.GONE);
1146 }
1147 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1148 if (remoteVideoTrack.isPresent()) {
1149 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1150 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1151 binding.remoteVideo.setScalingType(
1152 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1153 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1154 if (state == RtpEndUserState.CONNECTED) {
1155 binding.appBarLayout.setVisibility(View.GONE);
1156 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1157 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1158 } else {
1159 binding.appBarLayout.setVisibility(View.VISIBLE);
1160 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1161 binding.remoteVideoWrapper.setVisibility(View.GONE);
1162 }
1163 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1164 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1165 } else {
1166 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1167 }
1168 } else {
1169 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1170 binding.remoteVideoWrapper.setVisibility(View.GONE);
1171 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1172 }
1173 }
1174
1175 private Optional<VideoTrack> getLocalVideoTrack() {
1176 final JingleRtpConnection connection =
1177 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1178 if (connection == null) {
1179 return Optional.absent();
1180 }
1181 return connection.getLocalVideoTrack();
1182 }
1183
1184 private Optional<VideoTrack> getRemoteVideoTrack() {
1185 final JingleRtpConnection connection =
1186 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1187 if (connection == null) {
1188 return Optional.absent();
1189 }
1190 return connection.getRemoteVideoTrack();
1191 }
1192
1193 private void disableMicrophone(View view) {
1194 final JingleRtpConnection rtpConnection = requireRtpConnection();
1195 if (rtpConnection.setMicrophoneEnabled(false)) {
1196 updateInCallButtonConfiguration();
1197 }
1198 }
1199
1200 private void enableMicrophone(View view) {
1201 final JingleRtpConnection rtpConnection = requireRtpConnection();
1202 if (rtpConnection.setMicrophoneEnabled(true)) {
1203 updateInCallButtonConfiguration();
1204 }
1205 }
1206
1207 private void switchToEarpiece(View view) {
1208 requireRtpConnection()
1209 .getAudioManager()
1210 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1211 acquireProximityWakeLock();
1212 }
1213
1214 private void switchToSpeaker(View view) {
1215 requireRtpConnection()
1216 .getAudioManager()
1217 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1218 releaseProximityWakeLock();
1219 }
1220
1221 private void retry(View view) {
1222 final Intent intent = getIntent();
1223 final Account account = extractAccount(intent);
1224 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1225 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1226 final String action = intent.getAction();
1227 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1228 this.rtpConnectionReference = null;
1229 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1230 proposeJingleRtpSession(account, with, media);
1231 }
1232
1233 private void exit(final View view) {
1234 finish();
1235 }
1236
1237 private void recordVoiceMail(final View view) {
1238 final Intent intent = getIntent();
1239 final Account account = extractAccount(intent);
1240 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1241 final Conversation conversation =
1242 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1243 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1244 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1245 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1246 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1247 launchIntent.putExtra(
1248 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1249 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1250 startActivity(launchIntent);
1251 finish();
1252 }
1253
1254 private Contact getWith() {
1255 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1256 final Account account = id.account;
1257 return account.getRoster().getContact(id.with);
1258 }
1259
1260 private JingleRtpConnection requireRtpConnection() {
1261 final JingleRtpConnection connection =
1262 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1263 if (connection == null) {
1264 throw new IllegalStateException("No RTP connection found");
1265 }
1266 return connection;
1267 }
1268
1269 @Override
1270 public void onJingleRtpConnectionUpdate(
1271 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1272 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1273 if (END_CARD.contains(state)) {
1274 Log.d(Config.LOGTAG, "end card reached");
1275 releaseProximityWakeLock();
1276 runOnUiThread(
1277 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1278 }
1279 if (with.isBareJid()) {
1280 updateRtpSessionProposalState(account, with, state);
1281 return;
1282 }
1283 if (emptyReference(this.rtpConnectionReference)) {
1284 if (END_CARD.contains(state)) {
1285 Log.d(Config.LOGTAG, "not reinitializing session");
1286 return;
1287 }
1288 // this happens when going from proposed session to actual session
1289 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1290 return;
1291 }
1292 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1293 final boolean verified = requireRtpConnection().isVerified();
1294 final Set<Media> media = getMedia();
1295 final Contact contact = getWith();
1296 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1297 if (state == RtpEndUserState.ENDED) {
1298 finish();
1299 return;
1300 }
1301 runOnUiThread(
1302 () -> {
1303 updateStateDisplay(state, media);
1304 updateVerifiedShield(
1305 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1306 updateButtonConfiguration(state, media);
1307 updateVideoViews(state);
1308 updateIncomingCallScreen(state, contact);
1309 invalidateOptionsMenu();
1310 });
1311 if (END_CARD.contains(state)) {
1312 final JingleRtpConnection rtpConnection = requireRtpConnection();
1313 resetIntent(account, with, state, rtpConnection.getMedia());
1314 releaseVideoTracks(rtpConnection);
1315 this.rtpConnectionReference = null;
1316 }
1317 } else {
1318 Log.d(Config.LOGTAG, "received update for other rtp session");
1319 }
1320 }
1321
1322 @Override
1323 public void onAudioDeviceChanged(
1324 AppRTCAudioManager.AudioDevice selectedAudioDevice,
1325 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1326 Log.d(
1327 Config.LOGTAG,
1328 "onAudioDeviceChanged in activity: selected:"
1329 + selectedAudioDevice
1330 + ", available:"
1331 + availableAudioDevices);
1332 try {
1333 if (getMedia().contains(Media.VIDEO)) {
1334 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1335 return;
1336 }
1337 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1338 if (endUserState == RtpEndUserState.CONNECTED) {
1339 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1340 updateInCallButtonConfigurationSpeaker(
1341 audioManager.getSelectedAudioDevice(),
1342 audioManager.getAudioDevices().size());
1343 } else if (END_CARD.contains(endUserState)) {
1344 Log.d(
1345 Config.LOGTAG,
1346 "onAudioDeviceChanged() nothing to do because end card has been reached");
1347 } else {
1348 putProximityWakeLockInProperState(selectedAudioDevice);
1349 }
1350 } catch (IllegalStateException e) {
1351 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1352 }
1353 }
1354
1355 @Override
1356 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1357 super.onSaveInstanceState(outState);
1358 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1359 }
1360
1361 private void updateRtpSessionProposalState(
1362 final Account account, final Jid with, final RtpEndUserState state) {
1363 final Intent currentIntent = getIntent();
1364 final String withExtra =
1365 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1366 if (withExtra == null) {
1367 return;
1368 }
1369 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1370 runOnUiThread(
1371 () -> {
1372 updateVerifiedShield(false);
1373 updateStateDisplay(state);
1374 updateButtonConfiguration(state);
1375 updateIncomingCallScreen(state);
1376 invalidateOptionsMenu();
1377 });
1378 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1379 }
1380 }
1381
1382 private void resetIntent(final Bundle extras) {
1383 final Intent intent = new Intent(Intent.ACTION_VIEW);
1384 intent.putExtras(extras);
1385 setIntent(intent);
1386 }
1387
1388 private void resetIntent(
1389 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1390 final Intent intent = new Intent(Intent.ACTION_VIEW);
1391 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1392 if (account.getRoster()
1393 .getContact(with)
1394 .getPresences()
1395 .anySupport(Namespace.JINGLE_MESSAGE)) {
1396 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1397 } else {
1398 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1399 }
1400 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1401 intent.putExtra(
1402 EXTRA_LAST_ACTION,
1403 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1404 setIntent(intent);
1405 }
1406
1407 private static boolean emptyReference(final WeakReference<?> weakReference) {
1408 return weakReference == null || weakReference.get() == null;
1409 }
1410}