1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.PackageManager;
14import android.os.Build;
15import android.os.Bundle;
16import android.os.Handler;
17import android.os.PowerManager;
18import android.os.SystemClock;
19import android.util.Log;
20import android.util.Rational;
21import android.view.KeyEvent;
22import android.view.Menu;
23import android.view.MenuItem;
24import android.view.View;
25import android.view.WindowManager;
26import android.widget.Toast;
27
28import androidx.annotation.NonNull;
29import androidx.annotation.Nullable;
30import androidx.annotation.RequiresApi;
31import androidx.annotation.StringRes;
32import androidx.databinding.DataBindingUtil;
33
34import com.google.common.base.Optional;
35import com.google.common.base.Preconditions;
36import com.google.common.base.Throwables;
37import com.google.common.collect.ImmutableList;
38import com.google.common.collect.ImmutableSet;
39import com.google.common.util.concurrent.FutureCallback;
40import com.google.common.util.concurrent.Futures;
41
42import org.webrtc.RendererCommon;
43import org.webrtc.SurfaceViewRenderer;
44import org.webrtc.VideoTrack;
45
46import java.lang.ref.WeakReference;
47import java.util.Arrays;
48import java.util.Collections;
49import java.util.List;
50import java.util.Set;
51
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.R;
54import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.services.AppRTCAudioManager;
59import eu.siacs.conversations.services.XmppConnectionService;
60import eu.siacs.conversations.ui.util.AvatarWorkerTask;
61import eu.siacs.conversations.ui.util.MainThreadExecutor;
62import eu.siacs.conversations.ui.util.Rationals;
63import eu.siacs.conversations.utils.PermissionUtils;
64import eu.siacs.conversations.utils.TimeFrameUtils;
65import eu.siacs.conversations.xml.Namespace;
66import eu.siacs.conversations.xmpp.Jid;
67import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
68import eu.siacs.conversations.xmpp.jingle.ContentAddition;
69import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
70import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
71import eu.siacs.conversations.xmpp.jingle.Media;
72import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
73
74public class RtpSessionActivity extends XmppActivity
75 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
76 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
77
78 public static final String EXTRA_WITH = "with";
79 public static final String EXTRA_SESSION_ID = "session_id";
80 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
81 public static final String EXTRA_LAST_ACTION = "last_action";
82 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
83 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
84 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
85
86 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
87
88 private static final List<RtpEndUserState> END_CARD =
89 Arrays.asList(
90 RtpEndUserState.APPLICATION_ERROR,
91 RtpEndUserState.SECURITY_ERROR,
92 RtpEndUserState.DECLINED_OR_BUSY,
93 RtpEndUserState.CONNECTIVITY_ERROR,
94 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
95 RtpEndUserState.RETRACTED);
96 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
97 Arrays.asList(
98 RtpEndUserState.APPLICATION_ERROR,
99 RtpEndUserState.CONNECTIVITY_ERROR,
100 RtpEndUserState.SECURITY_ERROR);
101 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
102 Arrays.asList(
103 RtpEndUserState.CONNECTING,
104 RtpEndUserState.CONNECTED,
105 RtpEndUserState.RECONNECTING,
106 RtpEndUserState.INCOMING_CONTENT_ADD);
107 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
108 Arrays.asList(
109 RtpEndUserState.CONNECTED,
110 RtpEndUserState.RECONNECTING);
111 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
112 Arrays.asList(
113 RtpEndUserState.ACCEPTING_CALL,
114 RtpEndUserState.CONNECTING,
115 RtpEndUserState.RECONNECTING);
116 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
117 private static final int REQUEST_ACCEPT_CALL = 0x1111;
118 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
119 private static final int REQUEST_ADD_CONTENT = 0x1113;
120 private WeakReference<JingleRtpConnection> rtpConnectionReference;
121
122 private ActivityRtpSessionBinding binding;
123 private PowerManager.WakeLock mProximityWakeLock;
124
125 private final Handler mHandler = new Handler();
126 private final Runnable mTickExecutor =
127 new Runnable() {
128 @Override
129 public void run() {
130 updateCallDuration();
131 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
132 }
133 };
134
135 private static Set<Media> actionToMedia(final String action) {
136 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
137 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
138 } else {
139 return ImmutableSet.of(Media.AUDIO);
140 }
141 }
142
143 private static void addSink(
144 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
145 try {
146 videoTrack.addSink(surfaceViewRenderer);
147 } catch (final IllegalStateException e) {
148 Log.e(
149 Config.LOGTAG,
150 "possible race condition on trying to display video track. ignoring",
151 e);
152 }
153 }
154
155 @Override
156 public void onCreate(Bundle savedInstanceState) {
157 super.onCreate(savedInstanceState);
158 getWindow()
159 .addFlags(
160 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
161 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
162 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
163 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
164 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
165 setSupportActionBar(binding.toolbar);
166 }
167
168 @Override
169 public boolean onCreateOptionsMenu(final Menu menu) {
170 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
171 final MenuItem help = menu.findItem(R.id.action_help);
172 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
173 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
174 help.setVisible(Config.HELP != null && isHelpButtonVisible());
175 gotoChat.setVisible(isSwitchToConversationVisible());
176 switchToVideo.setVisible(isSwitchToVideoVisible());
177 return super.onCreateOptionsMenu(menu);
178 }
179
180 @Override
181 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
182 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
183 if (xmppConnectionService != null) {
184 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
185 return true;
186 }
187 }
188 }
189 return super.onKeyDown(keyCode, event);
190 }
191
192 private boolean isHelpButtonVisible() {
193 try {
194 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
195 } catch (IllegalStateException e) {
196 final Intent intent = getIntent();
197 final String state =
198 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
199 if (state != null) {
200 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
201 } else {
202 return false;
203 }
204 }
205 }
206
207 private boolean isSwitchToConversationVisible() {
208 final JingleRtpConnection connection =
209 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
210 return connection != null
211 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
212 }
213
214 private boolean isSwitchToVideoVisible() {
215 final JingleRtpConnection connection =
216 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
217 if (connection == null) {
218 return false;
219 }
220 return Media.audioOnly(connection.getMedia()) && STATES_CONSIDERED_CONNECTED.contains(connection.getEndUserState());
221 }
222
223 private void switchToConversation() {
224 final Contact contact = getWith();
225 final Conversation conversation =
226 xmppConnectionService.findOrCreateConversation(
227 contact.getAccount(), contact.getJid(), false, true);
228 switchToConversation(conversation);
229 }
230
231 public boolean onOptionsItemSelected(final MenuItem item) {
232 switch (item.getItemId()) {
233 case R.id.action_help:
234 launchHelpInBrowser();
235 return true;
236 case R.id.action_goto_chat:
237 switchToConversation();
238 return true;
239 case R.id.action_switch_to_video:
240 requestPermissionAndSwitchToVideo();
241 return true;
242 }
243 return super.onOptionsItemSelected(item);
244 }
245
246 private void launchHelpInBrowser() {
247 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
248 try {
249 startActivity(intent);
250 } catch (final ActivityNotFoundException e) {
251 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
252 .show();
253 }
254 }
255
256 private void endCall(View view) {
257 endCall();
258 }
259
260 private void endCall() {
261 if (this.rtpConnectionReference == null) {
262 retractSessionProposal();
263 finish();
264 } else {
265 requireRtpConnection().endCall();
266 }
267 }
268
269 private void retractSessionProposal() {
270 final Intent intent = getIntent();
271 final String action = intent.getAction();
272 final Account account = extractAccount(intent);
273 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
274 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
275 if (!Intent.ACTION_VIEW.equals(action)
276 || state == null
277 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
278 resetIntent(
279 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
280 }
281 xmppConnectionService
282 .getJingleConnectionManager()
283 .retractSessionProposal(account, with.asBareJid());
284 }
285
286 private void rejectCall(View view) {
287 requireRtpConnection().rejectCall();
288 finish();
289 }
290
291 private void acceptCall(View view) {
292 requestPermissionsAndAcceptCall();
293 }
294
295 private void acceptContentAdd() {
296 try {
297 requireRtpConnection()
298 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
299 } catch (final IllegalStateException e) {
300 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
301 }
302 }
303
304 private void requestPermissionAndSwitchToVideo() {
305 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
306 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
307 switchToVideo();
308 }
309 }
310
311 private void switchToVideo() {
312 try {
313 requireRtpConnection().addMedia(Media.VIDEO);
314 } catch (final IllegalStateException e) {
315 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
316 }
317 }
318
319 private void acceptContentAdd(final ContentAddition contentAddition) {
320 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
321 Log.d(Config.LOGTAG,"ignore press on content-accept button");
322 return;
323 }
324 requestPermissionAndAcceptContentAdd(contentAddition);
325 }
326
327 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
328 final List<String> permissions = permissions(contentAddition.media());
329 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
330 requireRtpConnection().acceptContentAdd(contentAddition.summary);
331 }
332 }
333
334 private void rejectContentAdd(final View view) {
335 requireRtpConnection().rejectContentAdd();
336 }
337
338 private void requestPermissionsAndAcceptCall() {
339 final List<String> permissions = permissions(getMedia());
340 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
341 putScreenInCallMode();
342 checkRecorderAndAcceptCall();
343 }
344 }
345
346 private List<String> permissions(final Set<Media> media) {
347 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
348 if (media.contains(Media.VIDEO)) {
349 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
350 } else {
351 permissions.add(Manifest.permission.RECORD_AUDIO);
352 }
353 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
354 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
355 }
356 return permissions.build();
357 }
358
359 private void checkRecorderAndAcceptCall() {
360 checkMicrophoneAvailabilityAsync();
361 try {
362 requireRtpConnection().acceptCall();
363 } catch (final IllegalStateException e) {
364 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
365 }
366 }
367
368 private void checkMicrophoneAvailabilityAsync() {
369 new Thread(new MicrophoneAvailabilityCheck(this)).start();
370 }
371
372 private static class MicrophoneAvailabilityCheck implements Runnable {
373
374 private final WeakReference<Activity> activityReference;
375
376 private MicrophoneAvailabilityCheck(final Activity activity) {
377 this.activityReference = new WeakReference<>(activity);
378 }
379
380 @Override
381 public void run() {
382 final long start = SystemClock.elapsedRealtime();
383 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
384 final long stop = SystemClock.elapsedRealtime();
385 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
386 if (isMicrophoneAvailable) {
387 return;
388 }
389 final Activity activity = activityReference.get();
390 if (activity == null) {
391 return;
392 }
393 activity.runOnUiThread(
394 () ->
395 Toast.makeText(
396 activity,
397 R.string.microphone_unavailable,
398 Toast.LENGTH_LONG)
399 .show());
400 }
401 }
402
403 private void putScreenInCallMode() {
404 putScreenInCallMode(requireRtpConnection().getMedia());
405 }
406
407 private void putScreenInCallMode(final Set<Media> media) {
408 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
409 if (!media.contains(Media.VIDEO)) {
410 final JingleRtpConnection rtpConnection =
411 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
412 final AppRTCAudioManager audioManager =
413 rtpConnection == null ? null : rtpConnection.getAudioManager();
414 if (audioManager == null
415 || audioManager.getSelectedAudioDevice()
416 == AppRTCAudioManager.AudioDevice.EARPIECE) {
417 acquireProximityWakeLock();
418 }
419 }
420 }
421
422 @SuppressLint("WakelockTimeout")
423 private void acquireProximityWakeLock() {
424 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
425 if (powerManager == null) {
426 Log.e(Config.LOGTAG, "power manager not available");
427 return;
428 }
429 if (isFinishing()) {
430 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
431 return;
432 }
433 if (this.mProximityWakeLock == null) {
434 this.mProximityWakeLock =
435 powerManager.newWakeLock(
436 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
437 }
438 if (!this.mProximityWakeLock.isHeld()) {
439 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
440 this.mProximityWakeLock.acquire();
441 }
442 }
443
444 private void releaseProximityWakeLock() {
445 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
446 Log.d(Config.LOGTAG, "releasing proximity wake lock");
447 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
448 this.mProximityWakeLock = null;
449 }
450 }
451
452 private void putProximityWakeLockInProperState(
453 final AppRTCAudioManager.AudioDevice audioDevice) {
454 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
455 acquireProximityWakeLock();
456 } else {
457 releaseProximityWakeLock();
458 }
459 }
460
461 @Override
462 protected void refreshUiReal() {}
463
464 @Override
465 public void onNewIntent(final Intent intent) {
466 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
467 super.onNewIntent(intent);
468 setIntent(intent);
469 if (xmppConnectionService == null) {
470 Log.d(
471 Config.LOGTAG,
472 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
473 return;
474 }
475 final Account account = extractAccount(intent);
476 final String action = intent.getAction();
477 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
478 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
479 if (sessionId != null) {
480 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
481 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
482 return;
483 }
484 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
485 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
486 requestPermissionsAndAcceptCall();
487 resetIntent(intent.getExtras());
488 }
489 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
490 proposeJingleRtpSession(account, with, actionToMedia(action));
491 setWith(account.getRoster().getContact(with), null);
492 } else {
493 throw new IllegalStateException("received onNewIntent without sessionId");
494 }
495 }
496
497 @Override
498 void onBackendConnected() {
499 final Intent intent = getIntent();
500 final String action = intent.getAction();
501 final Account account = extractAccount(intent);
502 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
503 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
504 if (sessionId != null) {
505 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
506 return;
507 }
508 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
509 Log.d(Config.LOGTAG, "intent action was accept");
510 requestPermissionsAndAcceptCall();
511 resetIntent(intent.getExtras());
512 }
513 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
514 proposeJingleRtpSession(account, with, actionToMedia(action));
515 setWith(account.getRoster().getContact(with), null);
516 } else if (Intent.ACTION_VIEW.equals(action)) {
517 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
518 final RtpEndUserState state =
519 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
520 if (state != null) {
521 Log.d(Config.LOGTAG, "restored last state from intent extra");
522 updateButtonConfiguration(state);
523 updateVerifiedShield(false);
524 updateStateDisplay(state);
525 updateIncomingCallScreen(state);
526 invalidateOptionsMenu();
527 }
528 setWith(account.getRoster().getContact(with), state);
529 if (xmppConnectionService
530 .getJingleConnectionManager()
531 .fireJingleRtpConnectionStateUpdates()) {
532 return;
533 }
534 if (END_CARD.contains(state)
535 || xmppConnectionService
536 .getJingleConnectionManager()
537 .hasMatchingProposal(account, with)) {
538 return;
539 }
540 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
541 finish();
542 }
543 }
544
545 private void setWidth(final RtpEndUserState state) {
546 setWith(getWith(), state);
547 }
548
549 private void setWith(final Contact contact, final RtpEndUserState state) {
550 binding.with.setText(contact.getDisplayName());
551 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
552 .contains(state)) {
553 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
554 binding.withJid.setVisibility(View.VISIBLE);
555 } else {
556 binding.withJid.setVisibility(View.GONE);
557 }
558 }
559
560 private void proposeJingleRtpSession(
561 final Account account, final Jid with, final Set<Media> media) {
562 checkMicrophoneAvailabilityAsync();
563 if (with.isBareJid()) {
564 xmppConnectionService
565 .getJingleConnectionManager()
566 .proposeJingleRtpSession(account, with, media);
567 } else {
568 final String sessionId =
569 xmppConnectionService
570 .getJingleConnectionManager()
571 .initializeRtpSession(account, with, media);
572 initializeActivityWithRunningRtpSession(account, with, sessionId);
573 resetIntent(account, with, sessionId);
574 }
575 putScreenInCallMode(media);
576 }
577
578 @Override
579 public void onRequestPermissionsResult(
580 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
581 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
582 final PermissionUtils.PermissionResult permissionResult =
583 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
584 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
585 if (requestCode == REQUEST_ACCEPT_CALL) {
586 checkRecorderAndAcceptCall();
587 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
588 acceptContentAdd();
589 } else if (requestCode == REQUEST_ADD_CONTENT) {
590 switchToVideo();
591 }
592 } else {
593 @StringRes int res;
594 final String firstDenied =
595 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
596 if (firstDenied == null) {
597 return;
598 }
599 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
600 res = R.string.no_microphone_permission;
601 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
602 res = R.string.no_camera_permission;
603 } else {
604 throw new IllegalStateException("Invalid permission result request");
605 }
606 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
607 .show();
608 }
609 }
610
611 @Override
612 public void onStart() {
613 super.onStart();
614 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
615 this.binding.remoteVideo.setOnAspectRatioChanged(this);
616 }
617
618 @Override
619 public void onStop() {
620 mHandler.removeCallbacks(mTickExecutor);
621 binding.remoteVideo.release();
622 binding.remoteVideo.setOnAspectRatioChanged(null);
623 binding.localVideo.release();
624 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
625 final JingleRtpConnection jingleRtpConnection =
626 weakReference == null ? null : weakReference.get();
627 if (jingleRtpConnection != null) {
628 releaseVideoTracks(jingleRtpConnection);
629 }
630 releaseProximityWakeLock();
631 super.onStop();
632 }
633
634 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
635 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
636 if (remoteVideo.isPresent()) {
637 remoteVideo.get().removeSink(binding.remoteVideo);
638 }
639 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
640 if (localVideo.isPresent()) {
641 localVideo.get().removeSink(binding.localVideo);
642 }
643 }
644
645 @Override
646 public void onBackPressed() {
647 if (isConnected()) {
648 if (switchToPictureInPicture()) {
649 return;
650 }
651 } else {
652 endCall();
653 }
654 super.onBackPressed();
655 }
656
657 @Override
658 public void onUserLeaveHint() {
659 super.onUserLeaveHint();
660 if (switchToPictureInPicture()) {
661 return;
662 }
663 // TODO apparently this method is not getting called on Android 10 when using the task
664 // switcher
665 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
666 retractSessionProposal();
667 }
668 }
669
670 private boolean isConnected() {
671 final JingleRtpConnection connection =
672 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
673 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
674 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
675 }
676
677 private boolean switchToPictureInPicture() {
678 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
679 if (shouldBePictureInPicture()) {
680 startPictureInPicture();
681 return true;
682 }
683 }
684 return false;
685 }
686
687 @RequiresApi(api = Build.VERSION_CODES.O)
688 private void startPictureInPicture() {
689 try {
690 final Rational rational = this.binding.remoteVideo.getAspectRatio();
691 final Rational clippedRational = Rationals.clip(rational);
692 Log.d(
693 Config.LOGTAG,
694 "suggested rational " + rational + ". clipped to " + clippedRational);
695 enterPictureInPictureMode(
696 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
697 } catch (final IllegalStateException e) {
698 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
699 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
700 }
701 }
702
703 @Override
704 public void onAspectRatioChanged(final Rational rational) {
705 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
706 final Rational clippedRational = Rationals.clip(rational);
707 Log.d(
708 Config.LOGTAG,
709 "suggested rational after aspect ratio change "
710 + rational
711 + ". clipped to "
712 + clippedRational);
713 setPictureInPictureParams(
714 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
715 }
716 }
717
718 private boolean deviceSupportsPictureInPicture() {
719 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
720 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
721 } else {
722 return false;
723 }
724 }
725
726 private boolean shouldBePictureInPicture() {
727 try {
728 final JingleRtpConnection rtpConnection = requireRtpConnection();
729 return rtpConnection.getMedia().contains(Media.VIDEO)
730 && Arrays.asList(
731 RtpEndUserState.ACCEPTING_CALL,
732 RtpEndUserState.CONNECTING,
733 RtpEndUserState.CONNECTED)
734 .contains(rtpConnection.getEndUserState());
735 } catch (final IllegalStateException e) {
736 return false;
737 }
738 }
739
740 private boolean initializeActivityWithRunningRtpSession(
741 final Account account, Jid with, String sessionId) {
742 final WeakReference<JingleRtpConnection> reference =
743 xmppConnectionService
744 .getJingleConnectionManager()
745 .findJingleRtpConnection(account, with, sessionId);
746 if (reference == null || reference.get() == null) {
747 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
748 xmppConnectionService
749 .getJingleConnectionManager()
750 .getTerminalSessionState(with, sessionId);
751 if (terminatedRtpSession == null) {
752 throw new IllegalStateException(
753 "failed to initialize activity with running rtp session. session not found");
754 }
755 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
756 return true;
757 }
758 this.rtpConnectionReference = reference;
759 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
760 final boolean verified = requireRtpConnection().isVerified();
761 if (currentState == RtpEndUserState.ENDED) {
762 finish();
763 return true;
764 }
765 final Set<Media> media = getMedia();
766 final ContentAddition contentAddition = getPendingContentAddition();
767 if (currentState == RtpEndUserState.INCOMING_CALL) {
768 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
769 }
770 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
771 requireRtpConnection().getState())) {
772 putScreenInCallMode();
773 }
774 setWidth(currentState);
775 updateVideoViews(currentState);
776 updateStateDisplay(currentState, media, contentAddition);
777 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
778 updateButtonConfiguration(currentState, media, contentAddition);
779 updateIncomingCallScreen(currentState);
780 invalidateOptionsMenu();
781 return false;
782 }
783
784 private void initializeWithTerminatedSessionState(
785 final Account account,
786 final Jid with,
787 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
788 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
789 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
790 finish();
791 return;
792 }
793 final RtpEndUserState state = terminatedRtpSession.state;
794 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
795 updateButtonConfiguration(state);
796 updateStateDisplay(state);
797 updateIncomingCallScreen(state);
798 updateCallDuration();
799 updateVerifiedShield(false);
800 invalidateOptionsMenu();
801 setWith(account.getRoster().getContact(with), state);
802 }
803
804 private void reInitializeActivityWithRunningRtpSession(
805 final Account account, Jid with, String sessionId) {
806 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
807 resetIntent(account, with, sessionId);
808 }
809
810 private void resetIntent(final Account account, final Jid with, final String sessionId) {
811 final Intent intent = new Intent(Intent.ACTION_VIEW);
812 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
813 intent.putExtra(EXTRA_WITH, with.toEscapedString());
814 intent.putExtra(EXTRA_SESSION_ID, sessionId);
815 setIntent(intent);
816 }
817
818 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
819 surfaceViewRenderer.setVisibility(View.VISIBLE);
820 try {
821 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
822 } catch (final IllegalStateException e) {
823 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
824 }
825 surfaceViewRenderer.setEnableHardwareScaler(true);
826 }
827
828 private void updateStateDisplay(final RtpEndUserState state) {
829 updateStateDisplay(state, Collections.emptySet(), null);
830 }
831
832 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
833 switch (state) {
834 case INCOMING_CALL:
835 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
836 if (media.contains(Media.VIDEO)) {
837 setTitle(R.string.rtp_state_incoming_video_call);
838 } else {
839 setTitle(R.string.rtp_state_incoming_call);
840 }
841 break;
842 case INCOMING_CONTENT_ADD:
843 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
844 setTitle(R.string.rtp_state_content_add_video);
845 } else {
846 setTitle(R.string.rtp_state_content_add);
847 }
848 break;
849 case CONNECTING:
850 setTitle(R.string.rtp_state_connecting);
851 break;
852 case CONNECTED:
853 setTitle(R.string.rtp_state_connected);
854 break;
855 case RECONNECTING:
856 setTitle(R.string.rtp_state_reconnecting);
857 break;
858 case ACCEPTING_CALL:
859 setTitle(R.string.rtp_state_accepting_call);
860 break;
861 case ENDING_CALL:
862 setTitle(R.string.rtp_state_ending_call);
863 break;
864 case FINDING_DEVICE:
865 setTitle(R.string.rtp_state_finding_device);
866 break;
867 case RINGING:
868 setTitle(R.string.rtp_state_ringing);
869 break;
870 case DECLINED_OR_BUSY:
871 setTitle(R.string.rtp_state_declined_or_busy);
872 break;
873 case CONNECTIVITY_ERROR:
874 setTitle(R.string.rtp_state_connectivity_error);
875 break;
876 case CONNECTIVITY_LOST_ERROR:
877 setTitle(R.string.rtp_state_connectivity_lost_error);
878 break;
879 case RETRACTED:
880 setTitle(R.string.rtp_state_retracted);
881 break;
882 case APPLICATION_ERROR:
883 setTitle(R.string.rtp_state_application_failure);
884 break;
885 case SECURITY_ERROR:
886 setTitle(R.string.rtp_state_security_error);
887 break;
888 case ENDED:
889 throw new IllegalStateException(
890 "Activity should have called finishAndReleaseWakeLock();");
891 default:
892 throw new IllegalStateException(
893 String.format("State %s has not been handled in UI", state));
894 }
895 }
896
897 private void updateVerifiedShield(final boolean verified) {
898 if (isPictureInPicture()) {
899 this.binding.verified.setVisibility(View.GONE);
900 return;
901 }
902 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
903 }
904
905 private void updateIncomingCallScreen(final RtpEndUserState state) {
906 updateIncomingCallScreen(state, null);
907 }
908
909 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
910 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
911 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
912 if (show) {
913 binding.contactPhoto.setVisibility(View.VISIBLE);
914 if (contact == null) {
915 AvatarWorkerTask.loadAvatar(
916 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
917 } else {
918 AvatarWorkerTask.loadAvatar(
919 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
920 }
921 } else {
922 binding.contactPhoto.setVisibility(View.GONE);
923 }
924 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
925 binding.usingAccount.setVisibility(View.VISIBLE);
926 binding.usingAccount.setText(
927 getString(
928 R.string.using_account,
929 account.getJid().asBareJid().toEscapedString()));
930 } else {
931 binding.usingAccount.setVisibility(View.GONE);
932 binding.contactPhoto.setVisibility(View.GONE);
933 }
934 }
935
936 private Set<Media> getMedia() {
937 return requireRtpConnection().getMedia();
938 }
939
940 public ContentAddition getPendingContentAddition() {
941 return requireRtpConnection().getPendingContentAddition();
942 }
943
944 private void updateButtonConfiguration(final RtpEndUserState state) {
945 updateButtonConfiguration(state, Collections.emptySet(), null);
946 }
947
948 @SuppressLint("RestrictedApi")
949 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
950 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
951 this.binding.rejectCall.setVisibility(View.INVISIBLE);
952 this.binding.endCall.setVisibility(View.INVISIBLE);
953 this.binding.acceptCall.setVisibility(View.INVISIBLE);
954 } else if (state == RtpEndUserState.INCOMING_CALL) {
955 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
956 this.binding.rejectCall.setOnClickListener(this::rejectCall);
957 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
958 this.binding.rejectCall.setVisibility(View.VISIBLE);
959 this.binding.endCall.setVisibility(View.INVISIBLE);
960 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
961 this.binding.acceptCall.setOnClickListener(this::acceptCall);
962 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
963 this.binding.acceptCall.setVisibility(View.VISIBLE);
964 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
965 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
966 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
967 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
968 this.binding.rejectCall.setVisibility(View.VISIBLE);
969 this.binding.endCall.setVisibility(View.INVISIBLE);
970 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
971 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
972 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
973 this.binding.acceptCall.setVisibility(View.VISIBLE);
974 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
975 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
976 this.binding.rejectCall.setOnClickListener(this::exit);
977 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
978 this.binding.rejectCall.setVisibility(View.VISIBLE);
979 this.binding.endCall.setVisibility(View.INVISIBLE);
980 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
981 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
982 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
983 this.binding.acceptCall.setVisibility(View.VISIBLE);
984 } else if (asList(
985 RtpEndUserState.CONNECTIVITY_ERROR,
986 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
987 RtpEndUserState.APPLICATION_ERROR,
988 RtpEndUserState.RETRACTED,
989 RtpEndUserState.SECURITY_ERROR)
990 .contains(state)) {
991 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
992 this.binding.rejectCall.setOnClickListener(this::exit);
993 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
994 this.binding.rejectCall.setVisibility(View.VISIBLE);
995 this.binding.endCall.setVisibility(View.INVISIBLE);
996 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
997 this.binding.acceptCall.setOnClickListener(this::retry);
998 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
999 this.binding.acceptCall.setVisibility(View.VISIBLE);
1000 } else {
1001 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1002 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1003 this.binding.endCall.setOnClickListener(this::endCall);
1004 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1005 this.binding.endCall.setVisibility(View.VISIBLE);
1006 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1007 }
1008 updateInCallButtonConfiguration(state, media);
1009 }
1010
1011 private boolean isPictureInPicture() {
1012 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1013 return isInPictureInPictureMode();
1014 } else {
1015 return false;
1016 }
1017 }
1018
1019 private void updateInCallButtonConfiguration() {
1020 updateInCallButtonConfiguration(
1021 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1022 }
1023
1024 @SuppressLint("RestrictedApi")
1025 private void updateInCallButtonConfiguration(
1026 final RtpEndUserState state, final Set<Media> media) {
1027 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1028 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1029 if (media.contains(Media.VIDEO)) {
1030 final JingleRtpConnection rtpConnection = requireRtpConnection();
1031 updateInCallButtonConfigurationVideo(
1032 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1033 } else {
1034 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1035 updateInCallButtonConfigurationSpeaker(
1036 audioManager.getSelectedAudioDevice(),
1037 audioManager.getAudioDevices().size());
1038 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1039 }
1040 if (media.contains(Media.AUDIO)) {
1041 updateInCallButtonConfigurationMicrophone(
1042 requireRtpConnection().isMicrophoneEnabled());
1043 } else {
1044 this.binding.inCallActionLeft.setVisibility(View.GONE);
1045 }
1046 } else {
1047 this.binding.inCallActionLeft.setVisibility(View.GONE);
1048 this.binding.inCallActionRight.setVisibility(View.GONE);
1049 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1050 }
1051 }
1052
1053 @SuppressLint("RestrictedApi")
1054 private void updateInCallButtonConfigurationSpeaker(
1055 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1056 switch (selectedAudioDevice) {
1057 case EARPIECE:
1058 this.binding.inCallActionRight.setImageResource(
1059 R.drawable.ic_volume_off_black_24dp);
1060 if (numberOfChoices >= 2) {
1061 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1062 } else {
1063 this.binding.inCallActionRight.setOnClickListener(null);
1064 this.binding.inCallActionRight.setClickable(false);
1065 }
1066 break;
1067 case WIRED_HEADSET:
1068 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1069 this.binding.inCallActionRight.setOnClickListener(null);
1070 this.binding.inCallActionRight.setClickable(false);
1071 break;
1072 case SPEAKER_PHONE:
1073 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1074 if (numberOfChoices >= 2) {
1075 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1076 } else {
1077 this.binding.inCallActionRight.setOnClickListener(null);
1078 this.binding.inCallActionRight.setClickable(false);
1079 }
1080 break;
1081 case BLUETOOTH:
1082 this.binding.inCallActionRight.setImageResource(
1083 R.drawable.ic_bluetooth_audio_black_24dp);
1084 this.binding.inCallActionRight.setOnClickListener(null);
1085 this.binding.inCallActionRight.setClickable(false);
1086 break;
1087 }
1088 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1089 }
1090
1091 @SuppressLint("RestrictedApi")
1092 private void updateInCallButtonConfigurationVideo(
1093 final boolean videoEnabled, final boolean isCameraSwitchable) {
1094 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1095 if (isCameraSwitchable) {
1096 this.binding.inCallActionFarRight.setImageResource(
1097 R.drawable.ic_flip_camera_android_black_24dp);
1098 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1099 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1100 } else {
1101 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1102 }
1103 if (videoEnabled) {
1104 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1105 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1106 } else {
1107 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1108 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1109 }
1110 }
1111
1112 private void switchCamera(final View view) {
1113 Futures.addCallback(
1114 requireRtpConnection().switchCamera(),
1115 new FutureCallback<Boolean>() {
1116 @Override
1117 public void onSuccess(@Nullable Boolean isFrontCamera) {
1118 binding.localVideo.setMirror(isFrontCamera);
1119 }
1120
1121 @Override
1122 public void onFailure(@NonNull final Throwable throwable) {
1123 Log.d(
1124 Config.LOGTAG,
1125 "could not switch camera",
1126 Throwables.getRootCause(throwable));
1127 Toast.makeText(
1128 RtpSessionActivity.this,
1129 R.string.could_not_switch_camera,
1130 Toast.LENGTH_LONG)
1131 .show();
1132 }
1133 },
1134 MainThreadExecutor.getInstance());
1135 }
1136
1137 private void enableVideo(View view) {
1138 try {
1139 requireRtpConnection().setVideoEnabled(true);
1140 } catch (final IllegalStateException e) {
1141 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1142 return;
1143 }
1144 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1145 }
1146
1147 private void disableVideo(View view) {
1148 final JingleRtpConnection rtpConnection = requireRtpConnection();
1149 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1150 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1151 rtpConnection.retractContentAdd();
1152 return;
1153 }
1154 requireRtpConnection().setVideoEnabled(false);
1155 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1156 }
1157
1158 @SuppressLint("RestrictedApi")
1159 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1160 if (microphoneEnabled) {
1161 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1162 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1163 } else {
1164 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1165 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1166 }
1167 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1168 }
1169
1170 private void updateCallDuration() {
1171 final JingleRtpConnection connection =
1172 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1173 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1174 this.binding.duration.setVisibility(View.GONE);
1175 return;
1176 }
1177 if (connection.zeroDuration()) {
1178 this.binding.duration.setVisibility(View.GONE);
1179 } else {
1180 this.binding.duration.setText(
1181 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1182 this.binding.duration.setVisibility(View.VISIBLE);
1183 }
1184 }
1185
1186 private void updateVideoViews(final RtpEndUserState state) {
1187 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1188 binding.localVideo.setVisibility(View.GONE);
1189 binding.localVideo.release();
1190 binding.remoteVideoWrapper.setVisibility(View.GONE);
1191 binding.remoteVideo.release();
1192 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1193 if (isPictureInPicture()) {
1194 binding.appBarLayout.setVisibility(View.GONE);
1195 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1196 if (Arrays.asList(
1197 RtpEndUserState.APPLICATION_ERROR,
1198 RtpEndUserState.CONNECTIVITY_ERROR,
1199 RtpEndUserState.SECURITY_ERROR)
1200 .contains(state)) {
1201 binding.pipWarning.setVisibility(View.VISIBLE);
1202 binding.pipWaiting.setVisibility(View.GONE);
1203 } else {
1204 binding.pipWarning.setVisibility(View.GONE);
1205 binding.pipWaiting.setVisibility(View.GONE);
1206 }
1207 } else {
1208 binding.appBarLayout.setVisibility(View.VISIBLE);
1209 binding.pipPlaceholder.setVisibility(View.GONE);
1210 }
1211 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1212 return;
1213 }
1214 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1215 binding.localVideo.setVisibility(View.GONE);
1216 binding.remoteVideoWrapper.setVisibility(View.GONE);
1217 binding.appBarLayout.setVisibility(View.GONE);
1218 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1219 binding.pipWarning.setVisibility(View.GONE);
1220 binding.pipWaiting.setVisibility(View.VISIBLE);
1221 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1222 return;
1223 }
1224 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1225 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1226 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1227 // paint local view over remote view
1228 binding.localVideo.setZOrderMediaOverlay(true);
1229 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1230 addSink(localVideoTrack.get(), binding.localVideo);
1231 } else {
1232 binding.localVideo.setVisibility(View.GONE);
1233 }
1234 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1235 if (remoteVideoTrack.isPresent()) {
1236 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1237 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1238 binding.remoteVideo.setScalingType(
1239 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1240 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1241 if (state == RtpEndUserState.CONNECTED) {
1242 binding.appBarLayout.setVisibility(View.GONE);
1243 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1244 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1245 } else {
1246 binding.appBarLayout.setVisibility(View.VISIBLE);
1247 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1248 binding.remoteVideoWrapper.setVisibility(View.GONE);
1249 }
1250 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1251 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1252 } else {
1253 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1254 }
1255 } else {
1256 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1257 binding.remoteVideoWrapper.setVisibility(View.GONE);
1258 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1259 }
1260 }
1261
1262 private Optional<VideoTrack> getLocalVideoTrack() {
1263 final JingleRtpConnection connection =
1264 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1265 if (connection == null) {
1266 return Optional.absent();
1267 }
1268 return connection.getLocalVideoTrack();
1269 }
1270
1271 private Optional<VideoTrack> getRemoteVideoTrack() {
1272 final JingleRtpConnection connection =
1273 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1274 if (connection == null) {
1275 return Optional.absent();
1276 }
1277 return connection.getRemoteVideoTrack();
1278 }
1279
1280 private void disableMicrophone(View view) {
1281 final JingleRtpConnection rtpConnection = requireRtpConnection();
1282 if (rtpConnection.setMicrophoneEnabled(false)) {
1283 updateInCallButtonConfiguration();
1284 }
1285 }
1286
1287 private void enableMicrophone(View view) {
1288 final JingleRtpConnection rtpConnection = requireRtpConnection();
1289 if (rtpConnection.setMicrophoneEnabled(true)) {
1290 updateInCallButtonConfiguration();
1291 }
1292 }
1293
1294 private void switchToEarpiece(View view) {
1295 requireRtpConnection()
1296 .getAudioManager()
1297 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1298 acquireProximityWakeLock();
1299 }
1300
1301 private void switchToSpeaker(View view) {
1302 requireRtpConnection()
1303 .getAudioManager()
1304 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1305 releaseProximityWakeLock();
1306 }
1307
1308 private void retry(View view) {
1309 final Intent intent = getIntent();
1310 final Account account = extractAccount(intent);
1311 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1312 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1313 final String action = intent.getAction();
1314 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1315 this.rtpConnectionReference = null;
1316 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1317 proposeJingleRtpSession(account, with, media);
1318 }
1319
1320 private void exit(final View view) {
1321 finish();
1322 }
1323
1324 private void recordVoiceMail(final View view) {
1325 final Intent intent = getIntent();
1326 final Account account = extractAccount(intent);
1327 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1328 final Conversation conversation =
1329 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1330 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1331 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1332 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1333 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1334 launchIntent.putExtra(
1335 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1336 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1337 startActivity(launchIntent);
1338 finish();
1339 }
1340
1341 private Contact getWith() {
1342 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1343 final Account account = id.account;
1344 return account.getRoster().getContact(id.with);
1345 }
1346
1347 private JingleRtpConnection requireRtpConnection() {
1348 final JingleRtpConnection connection =
1349 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1350 if (connection == null) {
1351 throw new IllegalStateException("No RTP connection found");
1352 }
1353 return connection;
1354 }
1355
1356 @Override
1357 public void onJingleRtpConnectionUpdate(
1358 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1359 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1360 if (END_CARD.contains(state)) {
1361 Log.d(Config.LOGTAG, "end card reached");
1362 releaseProximityWakeLock();
1363 runOnUiThread(
1364 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1365 }
1366 if (with.isBareJid()) {
1367 updateRtpSessionProposalState(account, with, state);
1368 return;
1369 }
1370 if (emptyReference(this.rtpConnectionReference)) {
1371 if (END_CARD.contains(state)) {
1372 Log.d(Config.LOGTAG, "not reinitializing session");
1373 return;
1374 }
1375 // this happens when going from proposed session to actual session
1376 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1377 return;
1378 }
1379 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1380 final boolean verified = requireRtpConnection().isVerified();
1381 final Set<Media> media = getMedia();
1382 final ContentAddition contentAddition = getPendingContentAddition();
1383 final Contact contact = getWith();
1384 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1385 if (state == RtpEndUserState.ENDED) {
1386 finish();
1387 return;
1388 }
1389 runOnUiThread(
1390 () -> {
1391 updateStateDisplay(state, media, contentAddition);
1392 updateVerifiedShield(
1393 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1394 updateButtonConfiguration(state, media, contentAddition);
1395 updateVideoViews(state);
1396 updateIncomingCallScreen(state, contact);
1397 invalidateOptionsMenu();
1398 });
1399 if (END_CARD.contains(state)) {
1400 final JingleRtpConnection rtpConnection = requireRtpConnection();
1401 resetIntent(account, with, state, rtpConnection.getMedia());
1402 releaseVideoTracks(rtpConnection);
1403 this.rtpConnectionReference = null;
1404 }
1405 } else {
1406 Log.d(Config.LOGTAG, "received update for other rtp session");
1407 }
1408 }
1409
1410 @Override
1411 public void onAudioDeviceChanged(
1412 AppRTCAudioManager.AudioDevice selectedAudioDevice,
1413 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1414 Log.d(
1415 Config.LOGTAG,
1416 "onAudioDeviceChanged in activity: selected:"
1417 + selectedAudioDevice
1418 + ", available:"
1419 + availableAudioDevices);
1420 try {
1421 if (getMedia().contains(Media.VIDEO)) {
1422 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1423 return;
1424 }
1425 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1426 if (endUserState == RtpEndUserState.CONNECTED) {
1427 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1428 updateInCallButtonConfigurationSpeaker(
1429 audioManager.getSelectedAudioDevice(),
1430 audioManager.getAudioDevices().size());
1431 } else if (END_CARD.contains(endUserState)) {
1432 Log.d(
1433 Config.LOGTAG,
1434 "onAudioDeviceChanged() nothing to do because end card has been reached");
1435 } else {
1436 putProximityWakeLockInProperState(selectedAudioDevice);
1437 }
1438 } catch (IllegalStateException e) {
1439 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1440 }
1441 }
1442
1443 private void updateRtpSessionProposalState(
1444 final Account account, final Jid with, final RtpEndUserState state) {
1445 final Intent currentIntent = getIntent();
1446 final String withExtra =
1447 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1448 if (withExtra == null) {
1449 return;
1450 }
1451 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1452 runOnUiThread(
1453 () -> {
1454 updateVerifiedShield(false);
1455 updateStateDisplay(state);
1456 updateButtonConfiguration(state);
1457 updateIncomingCallScreen(state);
1458 invalidateOptionsMenu();
1459 });
1460 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1461 }
1462 }
1463
1464 private void resetIntent(final Bundle extras) {
1465 final Intent intent = new Intent(Intent.ACTION_VIEW);
1466 intent.putExtras(extras);
1467 setIntent(intent);
1468 }
1469
1470 private void resetIntent(
1471 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1472 final Intent intent = new Intent(Intent.ACTION_VIEW);
1473 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1474 if (account.getRoster()
1475 .getContact(with)
1476 .getPresences()
1477 .anySupport(Namespace.JINGLE_MESSAGE)) {
1478 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1479 } else {
1480 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1481 }
1482 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1483 intent.putExtra(
1484 EXTRA_LAST_ACTION,
1485 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1486 setIntent(intent);
1487 }
1488
1489 private static boolean emptyReference(final WeakReference<?> weakReference) {
1490 return weakReference == null || weakReference.get() == null;
1491 }
1492}