1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.opengl.GLException;
17import android.os.Build;
18import android.os.Bundle;
19import android.os.Handler;
20import android.os.PowerManager;
21import android.os.SystemClock;
22import android.util.Log;
23import android.util.Rational;
24import android.view.KeyEvent;
25import android.view.Menu;
26import android.view.MenuItem;
27import android.view.View;
28import android.view.WindowManager;
29import android.widget.Toast;
30
31import androidx.annotation.NonNull;
32import androidx.annotation.Nullable;
33import androidx.annotation.RequiresApi;
34import androidx.annotation.StringRes;
35import androidx.databinding.DataBindingUtil;
36
37import com.google.common.base.Optional;
38import com.google.common.base.Preconditions;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import org.jetbrains.annotations.NotNull;
46import org.webrtc.RendererCommon;
47import org.webrtc.SurfaceViewRenderer;
48import org.webrtc.VideoTrack;
49
50import java.lang.ref.WeakReference;
51import java.util.Arrays;
52import java.util.Collections;
53import java.util.List;
54import java.util.Set;
55
56import eu.siacs.conversations.Config;
57import eu.siacs.conversations.R;
58import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
59import eu.siacs.conversations.entities.Account;
60import eu.siacs.conversations.entities.Contact;
61import eu.siacs.conversations.entities.Conversation;
62import eu.siacs.conversations.services.AppRTCAudioManager;
63import eu.siacs.conversations.services.XmppConnectionService;
64import eu.siacs.conversations.ui.widget.DialpadView;
65import eu.siacs.conversations.ui.util.AvatarWorkerTask;
66import eu.siacs.conversations.ui.util.MainThreadExecutor;
67import eu.siacs.conversations.ui.util.Rationals;
68import eu.siacs.conversations.utils.PermissionUtils;
69import eu.siacs.conversations.utils.TimeFrameUtils;
70import eu.siacs.conversations.xmpp.Jid;
71import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
72import eu.siacs.conversations.xmpp.jingle.ContentAddition;
73import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
74import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
75import eu.siacs.conversations.xmpp.jingle.Media;
76import eu.siacs.conversations.xmpp.jingle.RtpCapability;
77import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
78
79import org.webrtc.RendererCommon;
80import org.webrtc.SurfaceViewRenderer;
81import org.webrtc.VideoTrack;
82
83import java.lang.ref.WeakReference;
84import java.util.Arrays;
85import java.util.Collections;
86import java.util.List;
87import java.util.Set;
88
89public class RtpSessionActivity extends XmppActivity
90 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
91 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
92
93 public static final String EXTRA_WITH = "with";
94 public static final String EXTRA_SESSION_ID = "session_id";
95 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
96 public static final String EXTRA_LAST_ACTION = "last_action";
97 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
98 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
99 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
100
101 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
102
103 public static final List<RtpEndUserState> END_CARD =
104 Arrays.asList(
105 RtpEndUserState.APPLICATION_ERROR,
106 RtpEndUserState.SECURITY_ERROR,
107 RtpEndUserState.DECLINED_OR_BUSY,
108 RtpEndUserState.CONNECTIVITY_ERROR,
109 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
110 RtpEndUserState.RETRACTED);
111 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
112 Arrays.asList(
113 RtpEndUserState.APPLICATION_ERROR,
114 RtpEndUserState.CONNECTIVITY_ERROR,
115 RtpEndUserState.SECURITY_ERROR);
116 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
117 Arrays.asList(
118 RtpEndUserState.CONNECTING,
119 RtpEndUserState.CONNECTED,
120 RtpEndUserState.RECONNECTING,
121 RtpEndUserState.INCOMING_CONTENT_ADD);
122 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
123 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
124 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
125 Arrays.asList(
126 RtpEndUserState.ACCEPTING_CALL,
127 RtpEndUserState.CONNECTING,
128 RtpEndUserState.RECONNECTING);
129 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
130 private static final int REQUEST_ACCEPT_CALL = 0x1111;
131 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
132 private static final int REQUEST_ADD_CONTENT = 0x1113;
133 private WeakReference<JingleRtpConnection> rtpConnectionReference;
134
135 private ActivityRtpSessionBinding binding;
136 private PowerManager.WakeLock mProximityWakeLock;
137
138 private final Handler mHandler = new Handler();
139 private final Runnable mTickExecutor =
140 new Runnable() {
141 @Override
142 public void run() {
143 updateCallDuration();
144 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
145 }
146 };
147
148 private static Set<Media> actionToMedia(final String action) {
149 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
150 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
151 } else {
152 return ImmutableSet.of(Media.AUDIO);
153 }
154 }
155
156 private static void addSink(
157 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
158 try {
159 videoTrack.addSink(surfaceViewRenderer);
160 } catch (final IllegalStateException e) {
161 Log.e(
162 Config.LOGTAG,
163 "possible race condition on trying to display video track. ignoring",
164 e);
165 }
166 }
167
168 @Override
169 public void onCreate(Bundle savedInstanceState) {
170 super.onCreate(savedInstanceState);
171 getWindow()
172 .addFlags(
173 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
174 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
175 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
176 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
177 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
178 setSupportActionBar(binding.toolbar);
179
180 binding.dialpad.setClickConsumer(tag -> {
181 final JingleRtpConnection connection =
182 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
183 if (connection != null) connection.applyDtmfTone(tag);
184 });
185
186 if (savedInstanceState != null) {
187 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
188 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
189 }
190 }
191
192 @Override
193 public boolean onCreateOptionsMenu(final Menu menu) {
194 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
195 final MenuItem help = menu.findItem(R.id.action_help);
196 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
197 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
198 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
199 help.setVisible(Config.HELP != null && isHelpButtonVisible());
200 gotoChat.setVisible(isSwitchToConversationVisible());
201 switchToVideo.setVisible(isSwitchToVideoVisible());
202 dialpad.setVisible(isAudioOnlyConversation());
203 return super.onCreateOptionsMenu(menu);
204 }
205
206 @Override
207 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
208 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
209 if (xmppConnectionService != null) {
210 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
211 return true;
212 }
213 }
214 }
215 return super.onKeyDown(keyCode, event);
216 }
217
218 private boolean isHelpButtonVisible() {
219 try {
220 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
221 } catch (IllegalStateException e) {
222 final Intent intent = getIntent();
223 final String state =
224 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
225 if (state != null) {
226 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
227 } else {
228 return false;
229 }
230 }
231 }
232
233 private boolean isSwitchToConversationVisible() {
234 final JingleRtpConnection connection =
235 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
236 return connection != null
237 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
238 }
239
240 private boolean isAudioOnlyConversation() {
241 final JingleRtpConnection connection =
242 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
243
244 return connection != null && !connection.getMedia().contains(Media.VIDEO);
245 }
246
247 private boolean isSwitchToVideoVisible() {
248 final JingleRtpConnection connection =
249 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
250 if (connection == null) {
251 return false;
252 }
253 return connection.isSwitchToVideoAvailable();
254 }
255
256 private void switchToConversation() {
257 final Contact contact = getWith();
258 final Conversation conversation =
259 xmppConnectionService.findOrCreateConversation(
260 contact.getAccount(), contact.getJid(), false, true);
261 switchToConversation(conversation);
262 }
263
264 private void toggleDialpadVisibility() {
265 if (binding.dialpad.getVisibility() == View.VISIBLE) {
266 binding.dialpad.setVisibility(View.GONE);
267 }
268 else {
269 binding.dialpad.setVisibility(View.VISIBLE);
270 }
271 }
272
273 public boolean onOptionsItemSelected(final MenuItem item) {
274 final var itemItem = item.getItemId();
275 if (itemItem == R.id.action_help) {
276 launchHelpInBrowser();
277 return true;
278 } else if (itemItem == R.id.action_goto_chat) {
279 switchToConversation();
280 return true;
281 } else if (itemItem == R.id.action_dialpad) {
282 toggleDialpadVisibility();
283 return true;
284 } else if (itemItem == R.id.action_switch_to_video) {
285 requestPermissionAndSwitchToVideo();
286 return true;
287 } else {
288 return super.onOptionsItemSelected(item);
289 }
290 }
291
292 private void launchHelpInBrowser() {
293 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
294 try {
295 startActivity(intent);
296 } catch (final ActivityNotFoundException e) {
297 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
298 .show();
299 }
300 }
301
302 private void endCall(View view) {
303 endCall();
304 }
305
306 private void endCall() {
307 if (this.rtpConnectionReference == null) {
308 retractSessionProposal();
309 finish();
310 } else {
311 requireRtpConnection().endCall();
312 }
313 }
314
315 private void retractSessionProposal() {
316 final Intent intent = getIntent();
317 final String action = intent.getAction();
318 final Account account = extractAccount(intent);
319 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
320 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
321 if (!Intent.ACTION_VIEW.equals(action)
322 || state == null
323 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
324 resetIntent(
325 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
326 }
327 xmppConnectionService
328 .getJingleConnectionManager()
329 .retractSessionProposal(account, with.asBareJid());
330 }
331
332 private void rejectCall(View view) {
333 requireRtpConnection().rejectCall();
334 finish();
335 }
336
337 private void acceptCall(View view) {
338 requestPermissionsAndAcceptCall();
339 }
340
341 private void acceptContentAdd() {
342 try {
343 requireRtpConnection()
344 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
345 } catch (final IllegalStateException e) {
346 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
347 }
348 }
349
350 private void requestPermissionAndSwitchToVideo() {
351 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
352 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
353 switchToVideo();
354 }
355 }
356
357 private void switchToVideo() {
358 try {
359 requireRtpConnection().addMedia(Media.VIDEO);
360 } catch (final IllegalStateException e) {
361 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
362 }
363 }
364
365 private void acceptContentAdd(final ContentAddition contentAddition) {
366 if (contentAddition == null
367 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
368 Log.d(Config.LOGTAG, "ignore press on content-accept button");
369 return;
370 }
371 requestPermissionAndAcceptContentAdd(contentAddition);
372 }
373
374 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
375 final List<String> permissions = permissions(contentAddition.media());
376 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
377 try {
378 requireRtpConnection().acceptContentAdd(contentAddition.summary);
379 } catch (final IllegalStateException e) {
380 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
381 }
382 }
383 }
384
385 private void rejectContentAdd(final View view) {
386 requireRtpConnection().rejectContentAdd();
387 }
388
389 private void requestPermissionsAndAcceptCall() {
390 final List<String> permissions = permissions(getMedia());
391 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
392 putScreenInCallMode();
393 checkRecorderAndAcceptCall();
394 }
395 }
396
397 private List<String> permissions(final Set<Media> media) {
398 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
399 if (media.contains(Media.VIDEO)) {
400 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
401 } else {
402 permissions.add(Manifest.permission.RECORD_AUDIO);
403 }
404 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
405 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
406 }
407 return permissions.build();
408 }
409
410 private void checkRecorderAndAcceptCall() {
411 checkMicrophoneAvailabilityAsync();
412 try {
413 requireRtpConnection().acceptCall();
414 } catch (final IllegalStateException e) {
415 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
416 }
417 }
418
419 private void checkMicrophoneAvailabilityAsync() {
420 new Thread(new MicrophoneAvailabilityCheck(this)).start();
421 }
422
423 private static class MicrophoneAvailabilityCheck implements Runnable {
424
425 private final WeakReference<Activity> activityReference;
426
427 private MicrophoneAvailabilityCheck(final Activity activity) {
428 this.activityReference = new WeakReference<>(activity);
429 }
430
431 @Override
432 public void run() {
433 final long start = SystemClock.elapsedRealtime();
434 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
435 final long stop = SystemClock.elapsedRealtime();
436 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
437 if (isMicrophoneAvailable) {
438 return;
439 }
440 final Activity activity = activityReference.get();
441 if (activity == null) {
442 return;
443 }
444 activity.runOnUiThread(
445 () ->
446 Toast.makeText(
447 activity,
448 R.string.microphone_unavailable,
449 Toast.LENGTH_LONG)
450 .show());
451 }
452 }
453
454 private void putScreenInCallMode() {
455 putScreenInCallMode(requireRtpConnection().getMedia());
456 }
457
458 private void putScreenInCallMode(final Set<Media> media) {
459 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
460 if (Media.audioOnly(media)) {
461 final JingleRtpConnection rtpConnection =
462 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
463 final AppRTCAudioManager audioManager =
464 rtpConnection == null ? null : rtpConnection.getAudioManager();
465 if (audioManager == null
466 || audioManager.getSelectedAudioDevice()
467 == AppRTCAudioManager.AudioDevice.EARPIECE) {
468 acquireProximityWakeLock();
469 }
470 }
471 lockOrientation(media);
472 }
473
474 private void lockOrientation(final Set<Media> media) {
475 if (Media.audioOnly(media)) {
476 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
477 } else {
478 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
479 }
480 }
481
482 @SuppressLint("WakelockTimeout")
483 private void acquireProximityWakeLock() {
484 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
485 if (powerManager == null) {
486 Log.e(Config.LOGTAG, "power manager not available");
487 return;
488 }
489 if (isFinishing()) {
490 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
491 return;
492 }
493 if (this.mProximityWakeLock == null) {
494 this.mProximityWakeLock =
495 powerManager.newWakeLock(
496 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
497 }
498 if (!this.mProximityWakeLock.isHeld()) {
499 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
500 this.mProximityWakeLock.acquire();
501 }
502 }
503
504 private void releaseProximityWakeLock() {
505 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
506 Log.d(Config.LOGTAG, "releasing proximity wake lock");
507 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
508 this.mProximityWakeLock = null;
509 }
510 }
511
512 private void putProximityWakeLockInProperState(
513 final AppRTCAudioManager.AudioDevice audioDevice) {
514 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
515 acquireProximityWakeLock();
516 } else {
517 releaseProximityWakeLock();
518 }
519 }
520
521 @Override
522 protected void refreshUiReal() {}
523
524 @Override
525 public void onNewIntent(final Intent intent) {
526 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
527 super.onNewIntent(intent);
528 setIntent(intent);
529 if (xmppConnectionService == null) {
530 Log.d(
531 Config.LOGTAG,
532 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
533 return;
534 }
535 final Account account = extractAccount(intent);
536 final String action = intent.getAction();
537 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
538 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
539 if (sessionId != null) {
540 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
541 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
542 return;
543 }
544 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
545 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
546 requestPermissionsAndAcceptCall();
547 resetIntent(intent.getExtras());
548 }
549 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
550 proposeJingleRtpSession(account, with, actionToMedia(action));
551 setWith(account.getRoster().getContact(with), null);
552 } else {
553 throw new IllegalStateException("received onNewIntent without sessionId");
554 }
555 }
556
557 @Override
558 void onBackendConnected() {
559 final Intent intent = getIntent();
560 final String action = intent.getAction();
561 final Account account = extractAccount(intent);
562 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
563 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
564 if (sessionId != null) {
565 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
566 return;
567 }
568 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
569 Log.d(Config.LOGTAG, "intent action was accept");
570 requestPermissionsAndAcceptCall();
571 resetIntent(intent.getExtras());
572 }
573 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
574 proposeJingleRtpSession(account, with, actionToMedia(action));
575 setWith(account.getRoster().getContact(with), null);
576 } else if (Intent.ACTION_VIEW.equals(action)) {
577 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
578 final RtpEndUserState state =
579 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
580 if (state != null) {
581 Log.d(Config.LOGTAG, "restored last state from intent extra");
582 updateButtonConfiguration(state);
583 updateVerifiedShield(false);
584 updateStateDisplay(state);
585 updateIncomingCallScreen(state);
586 invalidateOptionsMenu();
587 }
588 setWith(account.getRoster().getContact(with), state);
589 if (xmppConnectionService
590 .getJingleConnectionManager()
591 .fireJingleRtpConnectionStateUpdates()) {
592 return;
593 }
594 if (END_CARD.contains(state)
595 || xmppConnectionService
596 .getJingleConnectionManager()
597 .hasMatchingProposal(account, with)) {
598 return;
599 }
600 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
601 finish();
602 }
603 }
604
605 private void setWith(final RtpEndUserState state) {
606 setWith(getWith(), state);
607 }
608
609 private void setWith(final Contact contact, final RtpEndUserState state) {
610 binding.with.setText(contact.getDisplayName());
611 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
612 .contains(state)) {
613 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
614 binding.withJid.setVisibility(View.VISIBLE);
615 } else {
616 binding.withJid.setVisibility(View.GONE);
617 }
618 }
619
620 private void proposeJingleRtpSession(
621 final Account account, final Jid with, final Set<Media> media) {
622 checkMicrophoneAvailabilityAsync();
623 if (with.isBareJid()) {
624 xmppConnectionService
625 .getJingleConnectionManager()
626 .proposeJingleRtpSession(account, with, media);
627 } else {
628 final String sessionId =
629 xmppConnectionService
630 .getJingleConnectionManager()
631 .initializeRtpSession(account, with, media);
632 initializeActivityWithRunningRtpSession(account, with, sessionId);
633 resetIntent(account, with, sessionId);
634 }
635 putScreenInCallMode(media);
636 }
637
638 @Override
639 public void onRequestPermissionsResult(
640 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
641 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
642 final PermissionUtils.PermissionResult permissionResult =
643 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
644 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
645 if (requestCode == REQUEST_ACCEPT_CALL) {
646 checkRecorderAndAcceptCall();
647 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
648 acceptContentAdd();
649 } else if (requestCode == REQUEST_ADD_CONTENT) {
650 switchToVideo();
651 }
652 } else {
653 @StringRes int res;
654 final String firstDenied =
655 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
656 if (firstDenied == null) {
657 return;
658 }
659 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
660 res = R.string.no_microphone_permission;
661 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
662 res = R.string.no_camera_permission;
663 } else {
664 throw new IllegalStateException("Invalid permission result request");
665 }
666 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
667 .show();
668 }
669 }
670
671 @Override
672 public void onStart() {
673 super.onStart();
674 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
675 this.binding.remoteVideo.setOnAspectRatioChanged(this);
676 }
677
678 @Override
679 public void onStop() {
680 mHandler.removeCallbacks(mTickExecutor);
681 binding.remoteVideo.release();
682 binding.remoteVideo.setOnAspectRatioChanged(null);
683 binding.localVideo.release();
684 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
685 final JingleRtpConnection jingleRtpConnection =
686 weakReference == null ? null : weakReference.get();
687 if (jingleRtpConnection != null) {
688 releaseVideoTracks(jingleRtpConnection);
689 }
690 releaseProximityWakeLock();
691 super.onStop();
692 }
693
694 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
695 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
696 if (remoteVideo.isPresent()) {
697 remoteVideo.get().removeSink(binding.remoteVideo);
698 }
699 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
700 if (localVideo.isPresent()) {
701 localVideo.get().removeSink(binding.localVideo);
702 }
703 }
704
705 @Override
706 public void onBackPressed() {
707 if (isConnected()) {
708 if (switchToPictureInPicture()) {
709 return;
710 }
711 } else {
712 endCall();
713 }
714 super.onBackPressed();
715 }
716
717 @Override
718 public void onUserLeaveHint() {
719 super.onUserLeaveHint();
720 if (switchToPictureInPicture()) {
721 return;
722 }
723 // TODO apparently this method is not getting called on Android 10 when using the task
724 // switcher
725 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
726 retractSessionProposal();
727 }
728 }
729
730 private boolean isConnected() {
731 final JingleRtpConnection connection =
732 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
733 final RtpEndUserState endUserState =
734 connection == null ? null : connection.getEndUserState();
735 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
736 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
737 }
738
739 private boolean switchToPictureInPicture() {
740 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
741 if (shouldBePictureInPicture()) {
742 startPictureInPicture();
743 return true;
744 }
745 }
746 return false;
747 }
748
749 @RequiresApi(api = Build.VERSION_CODES.O)
750 private void startPictureInPicture() {
751 try {
752 final Rational rational = this.binding.remoteVideo.getAspectRatio();
753 final Rational clippedRational = Rationals.clip(rational);
754 Log.d(
755 Config.LOGTAG,
756 "suggested rational " + rational + ". clipped to " + clippedRational);
757 enterPictureInPictureMode(
758 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
759 } catch (final IllegalStateException e) {
760 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
761 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
762 }
763 }
764
765 @Override
766 public void onAspectRatioChanged(final Rational rational) {
767 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
768 final Rational clippedRational = Rationals.clip(rational);
769 Log.d(
770 Config.LOGTAG,
771 "suggested rational after aspect ratio change "
772 + rational
773 + ". clipped to "
774 + clippedRational);
775 setPictureInPictureParams(
776 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
777 }
778 }
779
780 private boolean deviceSupportsPictureInPicture() {
781 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
782 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
783 } else {
784 return false;
785 }
786 }
787
788 private boolean shouldBePictureInPicture() {
789 try {
790 final JingleRtpConnection rtpConnection = requireRtpConnection();
791 return rtpConnection.getMedia().contains(Media.VIDEO)
792 && Arrays.asList(
793 RtpEndUserState.ACCEPTING_CALL,
794 RtpEndUserState.CONNECTING,
795 RtpEndUserState.CONNECTED)
796 .contains(rtpConnection.getEndUserState());
797 } catch (final IllegalStateException e) {
798 return false;
799 }
800 }
801
802 private boolean initializeActivityWithRunningRtpSession(
803 final Account account, Jid with, String sessionId) {
804 final WeakReference<JingleRtpConnection> reference =
805 xmppConnectionService
806 .getJingleConnectionManager()
807 .findJingleRtpConnection(account, with, sessionId);
808 if (reference == null || reference.get() == null) {
809 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
810 xmppConnectionService
811 .getJingleConnectionManager()
812 .getTerminalSessionState(with, sessionId);
813 if (terminatedRtpSession == null) {
814 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
815 finish();
816 return true;
817 }
818 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
819 return true;
820 }
821 this.rtpConnectionReference = reference;
822 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
823 final boolean verified = requireRtpConnection().isVerified();
824 if (currentState == RtpEndUserState.ENDED) {
825 finish();
826 return true;
827 }
828 final Set<Media> media = getMedia();
829 final ContentAddition contentAddition = getPendingContentAddition();
830 if (currentState == RtpEndUserState.INCOMING_CALL) {
831 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
832 }
833 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
834 requireRtpConnection().getState())) {
835 putScreenInCallMode();
836 }
837 setWith(currentState);
838 updateVideoViews(currentState);
839 updateStateDisplay(currentState, media, contentAddition);
840 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
841 updateButtonConfiguration(currentState, media, contentAddition);
842 updateIncomingCallScreen(currentState);
843 invalidateOptionsMenu();
844 return false;
845 }
846
847 private void initializeWithTerminatedSessionState(
848 final Account account,
849 final Jid with,
850 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
851 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
852 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
853 finish();
854 return;
855 }
856 final RtpEndUserState state = terminatedRtpSession.state;
857 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
858 updateButtonConfiguration(state);
859 updateStateDisplay(state);
860 updateIncomingCallScreen(state);
861 updateCallDuration();
862 updateVerifiedShield(false);
863 invalidateOptionsMenu();
864 setWith(account.getRoster().getContact(with), state);
865 }
866
867 private void reInitializeActivityWithRunningRtpSession(
868 final Account account, Jid with, String sessionId) {
869 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
870 resetIntent(account, with, sessionId);
871 }
872
873 private void resetIntent(final Account account, final Jid with, final String sessionId) {
874 final Intent intent = new Intent(Intent.ACTION_VIEW);
875 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
876 intent.putExtra(EXTRA_WITH, with.toEscapedString());
877 intent.putExtra(EXTRA_SESSION_ID, sessionId);
878 setIntent(intent);
879 }
880
881 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
882 surfaceViewRenderer.setVisibility(View.VISIBLE);
883 try {
884 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
885 } catch (final IllegalStateException ignored) {
886 // SurfaceViewRenderer was already initialized
887 } catch (final RuntimeException e) {
888 if (Throwables.getRootCause(e) instanceof GLException glException) {
889 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
890 }
891 }
892 surfaceViewRenderer.setEnableHardwareScaler(true);
893 }
894
895 private void updateStateDisplay(final RtpEndUserState state) {
896 updateStateDisplay(state, Collections.emptySet(), null);
897 }
898
899 private void updateStateDisplay(
900 final RtpEndUserState state,
901 final Set<Media> media,
902 final ContentAddition contentAddition) {
903 switch (state) {
904 case INCOMING_CALL -> {
905 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
906 if (media.contains(Media.VIDEO)) {
907 setTitle(R.string.rtp_state_incoming_video_call);
908 } else {
909 setTitle(R.string.rtp_state_incoming_call);
910 }
911 }
912 case INCOMING_CONTENT_ADD -> {
913 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
914 setTitle(R.string.rtp_state_content_add_video);
915 } else {
916 setTitle(R.string.rtp_state_content_add);
917 }
918 }
919 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
920 case CONNECTED -> setTitle(R.string.rtp_state_connected);
921 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
922 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
923 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
924 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
925 case RINGING -> setTitle(R.string.rtp_state_ringing);
926 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
927 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
928 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
929 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
930 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
931 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
932 case ENDED -> throw new IllegalStateException(
933 "Activity should have called finishAndReleaseWakeLock();");
934 default -> throw new IllegalStateException(
935 String.format("State %s has not been handled in UI", state));
936 }
937 }
938
939 private void updateVerifiedShield(final boolean verified) {
940 if (isPictureInPicture()) {
941 this.binding.verified.setVisibility(View.GONE);
942 return;
943 }
944 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
945 }
946
947 private void updateIncomingCallScreen(final RtpEndUserState state) {
948 updateIncomingCallScreen(state, null);
949 }
950
951 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
952 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
953 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
954 if (show) {
955 binding.contactPhoto.setVisibility(View.VISIBLE);
956 if (contact == null) {
957 AvatarWorkerTask.loadAvatar(
958 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
959 } else {
960 AvatarWorkerTask.loadAvatar(
961 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
962 }
963 } else {
964 binding.contactPhoto.setVisibility(View.GONE);
965 }
966 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
967 binding.usingAccount.setVisibility(View.VISIBLE);
968 binding.usingAccount.setText(
969 getString(
970 R.string.using_account,
971 account.getJid().asBareJid().toEscapedString()));
972 } else {
973 binding.usingAccount.setVisibility(View.GONE);
974 binding.contactPhoto.setVisibility(View.GONE);
975 }
976 }
977
978 private Set<Media> getMedia() {
979 return requireRtpConnection().getMedia();
980 }
981
982 public ContentAddition getPendingContentAddition() {
983 return requireRtpConnection().getPendingContentAddition();
984 }
985
986 private void updateButtonConfiguration(final RtpEndUserState state) {
987 updateButtonConfiguration(state, Collections.emptySet(), null);
988 }
989
990 @SuppressLint("RestrictedApi")
991 private void updateButtonConfiguration(
992 final RtpEndUserState state,
993 final Set<Media> media,
994 final ContentAddition contentAddition) {
995 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
996 this.binding.rejectCall.setVisibility(View.INVISIBLE);
997 this.binding.endCall.setVisibility(View.INVISIBLE);
998 this.binding.acceptCall.setVisibility(View.INVISIBLE);
999 } else if (state == RtpEndUserState.INCOMING_CALL) {
1000 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
1001 this.binding.rejectCall.setOnClickListener(this::rejectCall);
1002 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1003 this.binding.rejectCall.setVisibility(View.VISIBLE);
1004 this.binding.endCall.setVisibility(View.INVISIBLE);
1005 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
1006 this.binding.acceptCall.setOnClickListener(this::acceptCall);
1007 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
1008 this.binding.acceptCall.setVisibility(View.VISIBLE);
1009 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
1010 this.binding.rejectCall.setContentDescription(
1011 getString(R.string.reject_switch_to_video));
1012 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
1013 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1014 this.binding.rejectCall.setVisibility(View.VISIBLE);
1015 this.binding.endCall.setVisibility(View.INVISIBLE);
1016 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1017 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1018 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
1019 this.binding.acceptCall.setVisibility(View.VISIBLE);
1020 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
1021 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1022 this.binding.rejectCall.setOnClickListener(this::exit);
1023 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1024 this.binding.rejectCall.setVisibility(View.VISIBLE);
1025 this.binding.endCall.setVisibility(View.INVISIBLE);
1026 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1027 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1028 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1029 this.binding.acceptCall.setVisibility(View.VISIBLE);
1030 } else if (asList(
1031 RtpEndUserState.CONNECTIVITY_ERROR,
1032 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1033 RtpEndUserState.APPLICATION_ERROR,
1034 RtpEndUserState.RETRACTED,
1035 RtpEndUserState.SECURITY_ERROR)
1036 .contains(state)) {
1037 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1038 this.binding.rejectCall.setOnClickListener(this::exit);
1039 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1040 this.binding.rejectCall.setVisibility(View.VISIBLE);
1041 this.binding.endCall.setVisibility(View.INVISIBLE);
1042 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1043 this.binding.acceptCall.setOnClickListener(this::retry);
1044 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1045 this.binding.acceptCall.setVisibility(View.VISIBLE);
1046 } else {
1047 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1048 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1049 this.binding.endCall.setOnClickListener(this::endCall);
1050 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1051 this.binding.endCall.setVisibility(View.VISIBLE);
1052 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1053 }
1054 updateInCallButtonConfiguration(state, media);
1055 }
1056
1057 private boolean isPictureInPicture() {
1058 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1059 return isInPictureInPictureMode();
1060 } else {
1061 return false;
1062 }
1063 }
1064
1065 private void updateInCallButtonConfiguration() {
1066 updateInCallButtonConfiguration(
1067 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1068 }
1069
1070 @SuppressLint("RestrictedApi")
1071 private void updateInCallButtonConfiguration(
1072 final RtpEndUserState state, final Set<Media> media) {
1073 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1074 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1075 if (media.contains(Media.VIDEO)) {
1076 final JingleRtpConnection rtpConnection = requireRtpConnection();
1077 updateInCallButtonConfigurationVideo(
1078 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1079 } else {
1080 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1081 updateInCallButtonConfigurationSpeaker(
1082 audioManager.getSelectedAudioDevice(),
1083 audioManager.getAudioDevices().size());
1084 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1085 }
1086 if (media.contains(Media.AUDIO)) {
1087 updateInCallButtonConfigurationMicrophone(
1088 requireRtpConnection().isMicrophoneEnabled());
1089 } else {
1090 this.binding.inCallActionLeft.setVisibility(View.GONE);
1091 }
1092 } else {
1093 this.binding.inCallActionLeft.setVisibility(View.GONE);
1094 this.binding.inCallActionRight.setVisibility(View.GONE);
1095 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1096 }
1097 }
1098
1099 @SuppressLint("RestrictedApi")
1100 private void updateInCallButtonConfigurationSpeaker(
1101 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1102 switch (selectedAudioDevice) {
1103 case EARPIECE -> {
1104 this.binding.inCallActionRight.setImageResource(
1105 R.drawable.ic_volume_off_black_24dp);
1106 if (numberOfChoices >= 2) {
1107 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1108 } else {
1109 this.binding.inCallActionRight.setOnClickListener(null);
1110 this.binding.inCallActionRight.setClickable(false);
1111 }
1112 }
1113 case WIRED_HEADSET -> {
1114 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1115 this.binding.inCallActionRight.setOnClickListener(null);
1116 this.binding.inCallActionRight.setClickable(false);
1117 }
1118 case SPEAKER_PHONE -> {
1119 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1120 if (numberOfChoices >= 2) {
1121 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1122 } else {
1123 this.binding.inCallActionRight.setOnClickListener(null);
1124 this.binding.inCallActionRight.setClickable(false);
1125 }
1126 }
1127 case BLUETOOTH -> {
1128 this.binding.inCallActionRight.setImageResource(
1129 R.drawable.ic_bluetooth_audio_black_24dp);
1130 this.binding.inCallActionRight.setOnClickListener(null);
1131 this.binding.inCallActionRight.setClickable(false);
1132 }
1133 }
1134 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1135 }
1136
1137 @SuppressLint("RestrictedApi")
1138 private void updateInCallButtonConfigurationVideo(
1139 final boolean videoEnabled, final boolean isCameraSwitchable) {
1140 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1141 if (isCameraSwitchable) {
1142 this.binding.inCallActionFarRight.setImageResource(
1143 R.drawable.ic_flip_camera_android_black_24dp);
1144 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1145 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1146 } else {
1147 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1148 }
1149 if (videoEnabled) {
1150 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1151 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1152 } else {
1153 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1154 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1155 }
1156 }
1157
1158 private void switchCamera(final View view) {
1159 Futures.addCallback(
1160 requireRtpConnection().switchCamera(),
1161 new FutureCallback<>() {
1162 @Override
1163 public void onSuccess(@Nullable Boolean isFrontCamera) {
1164 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1165 }
1166
1167 @Override
1168 public void onFailure(@NonNull final Throwable throwable) {
1169 Log.d(
1170 Config.LOGTAG,
1171 "could not switch camera",
1172 Throwables.getRootCause(throwable));
1173 Toast.makeText(
1174 RtpSessionActivity.this,
1175 R.string.could_not_switch_camera,
1176 Toast.LENGTH_LONG)
1177 .show();
1178 }
1179 },
1180 MainThreadExecutor.getInstance());
1181 }
1182
1183 private void enableVideo(View view) {
1184 try {
1185 requireRtpConnection().setVideoEnabled(true);
1186 } catch (final IllegalStateException e) {
1187 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1188 return;
1189 }
1190 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1191 }
1192
1193 private void disableVideo(View view) {
1194 final JingleRtpConnection rtpConnection = requireRtpConnection();
1195 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1196 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1197 rtpConnection.retractContentAdd();
1198 return;
1199 }
1200 requireRtpConnection().setVideoEnabled(false);
1201 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1202 }
1203
1204 @SuppressLint("RestrictedApi")
1205 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1206 if (microphoneEnabled) {
1207 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1208 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1209 } else {
1210 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1211 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1212 }
1213 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1214 }
1215
1216 private void updateCallDuration() {
1217 final JingleRtpConnection connection =
1218 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1219 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1220 this.binding.duration.setVisibility(View.GONE);
1221 return;
1222 }
1223 if (connection.zeroDuration()) {
1224 this.binding.duration.setVisibility(View.GONE);
1225 } else {
1226 this.binding.duration.setText(
1227 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1228 this.binding.duration.setVisibility(View.VISIBLE);
1229 }
1230 }
1231
1232 private void updateVideoViews(final RtpEndUserState state) {
1233 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1234 binding.localVideo.setVisibility(View.GONE);
1235 binding.localVideo.release();
1236 binding.remoteVideoWrapper.setVisibility(View.GONE);
1237 binding.remoteVideo.release();
1238 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1239 if (isPictureInPicture()) {
1240 binding.appBarLayout.setVisibility(View.GONE);
1241 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1242 if (Arrays.asList(
1243 RtpEndUserState.APPLICATION_ERROR,
1244 RtpEndUserState.CONNECTIVITY_ERROR,
1245 RtpEndUserState.SECURITY_ERROR)
1246 .contains(state)) {
1247 binding.pipWarning.setVisibility(View.VISIBLE);
1248 binding.pipWaiting.setVisibility(View.GONE);
1249 } else {
1250 binding.pipWarning.setVisibility(View.GONE);
1251 binding.pipWaiting.setVisibility(View.GONE);
1252 }
1253 } else {
1254 binding.appBarLayout.setVisibility(View.VISIBLE);
1255 binding.pipPlaceholder.setVisibility(View.GONE);
1256 }
1257 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1258 return;
1259 }
1260 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1261 binding.localVideo.setVisibility(View.GONE);
1262 binding.remoteVideoWrapper.setVisibility(View.GONE);
1263 binding.appBarLayout.setVisibility(View.GONE);
1264 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1265 binding.pipWarning.setVisibility(View.GONE);
1266 binding.pipWaiting.setVisibility(View.VISIBLE);
1267 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1268 return;
1269 }
1270 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1271 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1272 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1273 // paint local view over remote view
1274 binding.localVideo.setZOrderMediaOverlay(true);
1275 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1276 addSink(localVideoTrack.get(), binding.localVideo);
1277 } else {
1278 binding.localVideo.setVisibility(View.GONE);
1279 }
1280 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1281 if (remoteVideoTrack.isPresent()) {
1282 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1283 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1284 binding.remoteVideo.setScalingType(
1285 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1286 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1287 if (state == RtpEndUserState.CONNECTED) {
1288 binding.appBarLayout.setVisibility(View.GONE);
1289 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1290 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1291 } else {
1292 binding.appBarLayout.setVisibility(View.VISIBLE);
1293 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1294 binding.remoteVideoWrapper.setVisibility(View.GONE);
1295 }
1296 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1297 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1298 } else {
1299 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1300 }
1301 } else {
1302 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1303 binding.remoteVideoWrapper.setVisibility(View.GONE);
1304 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1305 }
1306 }
1307
1308 private Optional<VideoTrack> getLocalVideoTrack() {
1309 final JingleRtpConnection connection =
1310 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1311 if (connection == null) {
1312 return Optional.absent();
1313 }
1314 return connection.getLocalVideoTrack();
1315 }
1316
1317 private Optional<VideoTrack> getRemoteVideoTrack() {
1318 final JingleRtpConnection connection =
1319 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1320 if (connection == null) {
1321 return Optional.absent();
1322 }
1323 return connection.getRemoteVideoTrack();
1324 }
1325
1326 private void disableMicrophone(View view) {
1327 final JingleRtpConnection rtpConnection = requireRtpConnection();
1328 if (rtpConnection.setMicrophoneEnabled(false)) {
1329 updateInCallButtonConfiguration();
1330 }
1331 }
1332
1333 private void enableMicrophone(View view) {
1334 final JingleRtpConnection rtpConnection = requireRtpConnection();
1335 if (rtpConnection.setMicrophoneEnabled(true)) {
1336 updateInCallButtonConfiguration();
1337 }
1338 }
1339
1340 private void switchToEarpiece(View view) {
1341 requireRtpConnection()
1342 .getAudioManager()
1343 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1344 acquireProximityWakeLock();
1345 }
1346
1347 private void switchToSpeaker(View view) {
1348 requireRtpConnection()
1349 .getAudioManager()
1350 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1351 releaseProximityWakeLock();
1352 }
1353
1354 private void retry(View view) {
1355 final Intent intent = getIntent();
1356 final Account account = extractAccount(intent);
1357 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1358 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1359 final String action = intent.getAction();
1360 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1361 this.rtpConnectionReference = null;
1362 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1363 proposeJingleRtpSession(account, with, media);
1364 }
1365
1366 private void exit(final View view) {
1367 finish();
1368 }
1369
1370 private void recordVoiceMail(final View view) {
1371 final Intent intent = getIntent();
1372 final Account account = extractAccount(intent);
1373 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1374 final Conversation conversation =
1375 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1376 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1377 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1378 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1379 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1380 launchIntent.putExtra(
1381 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1382 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1383 startActivity(launchIntent);
1384 finish();
1385 }
1386
1387 private Contact getWith() {
1388 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1389 final Account account = id.account;
1390 return account.getRoster().getContact(id.with);
1391 }
1392
1393 private JingleRtpConnection requireRtpConnection() {
1394 final JingleRtpConnection connection =
1395 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1396 if (connection == null) {
1397 throw new IllegalStateException("No RTP connection found");
1398 }
1399 return connection;
1400 }
1401
1402 @Override
1403 public void onJingleRtpConnectionUpdate(
1404 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1405 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1406 if (END_CARD.contains(state)) {
1407 Log.d(Config.LOGTAG, "end card reached");
1408 releaseProximityWakeLock();
1409 runOnUiThread(
1410 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1411 }
1412 if (with.isBareJid()) {
1413 updateRtpSessionProposalState(account, with, state);
1414 return;
1415 }
1416 if (emptyReference(this.rtpConnectionReference)) {
1417 if (END_CARD.contains(state)) {
1418 Log.d(Config.LOGTAG, "not reinitializing session");
1419 return;
1420 }
1421 // this happens when going from proposed session to actual session
1422 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1423 return;
1424 }
1425 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1426 final boolean verified = requireRtpConnection().isVerified();
1427 final Set<Media> media = getMedia();
1428 lockOrientation(media);
1429 final ContentAddition contentAddition = getPendingContentAddition();
1430 final Contact contact = getWith();
1431 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1432 if (state == RtpEndUserState.ENDED) {
1433 finish();
1434 return;
1435 }
1436 runOnUiThread(
1437 () -> {
1438 updateStateDisplay(state, media, contentAddition);
1439 updateVerifiedShield(
1440 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1441 updateButtonConfiguration(state, media, contentAddition);
1442 updateVideoViews(state);
1443 updateIncomingCallScreen(state, contact);
1444 invalidateOptionsMenu();
1445 });
1446 if (END_CARD.contains(state)) {
1447 final JingleRtpConnection rtpConnection = requireRtpConnection();
1448 resetIntent(account, with, state, rtpConnection.getMedia());
1449 releaseVideoTracks(rtpConnection);
1450 this.rtpConnectionReference = null;
1451 }
1452 } else {
1453 Log.d(Config.LOGTAG, "received update for other rtp session");
1454 }
1455 }
1456
1457 @Override
1458 public void onAudioDeviceChanged(
1459 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1460 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1461 Log.d(
1462 Config.LOGTAG,
1463 "onAudioDeviceChanged in activity: selected:"
1464 + selectedAudioDevice
1465 + ", available:"
1466 + availableAudioDevices);
1467 try {
1468 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1469 final Set<Media> media = getMedia();
1470 if (END_CARD.contains(endUserState)) {
1471 Log.d(
1472 Config.LOGTAG,
1473 "onAudioDeviceChanged() nothing to do because end card has been reached");
1474 } else {
1475 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1476 final AppRTCAudioManager audioManager =
1477 requireRtpConnection().getAudioManager();
1478 updateInCallButtonConfigurationSpeaker(
1479 audioManager.getSelectedAudioDevice(),
1480 audioManager.getAudioDevices().size());
1481 }
1482 Log.d(
1483 Config.LOGTAG,
1484 "put proximity wake lock into proper state after device update");
1485 putProximityWakeLockInProperState(selectedAudioDevice);
1486 }
1487 } catch (final IllegalStateException e) {
1488 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1489 }
1490 }
1491
1492 @Override
1493 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1494 super.onSaveInstanceState(outState);
1495 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1496 }
1497
1498 private void updateRtpSessionProposalState(
1499 final Account account, final Jid with, final RtpEndUserState state) {
1500 final Intent currentIntent = getIntent();
1501 final String withExtra =
1502 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1503 if (withExtra == null) {
1504 return;
1505 }
1506 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1507 runOnUiThread(
1508 () -> {
1509 updateVerifiedShield(false);
1510 updateStateDisplay(state);
1511 updateButtonConfiguration(state);
1512 updateIncomingCallScreen(state);
1513 invalidateOptionsMenu();
1514 });
1515 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1516 }
1517 }
1518
1519 private void resetIntent(final Bundle extras) {
1520 final Intent intent = new Intent(Intent.ACTION_VIEW);
1521 intent.putExtras(extras);
1522 setIntent(intent);
1523 }
1524
1525 private void resetIntent(
1526 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1527 final Intent intent = new Intent(Intent.ACTION_VIEW);
1528 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1529 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1530 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1531 } else {
1532 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1533 }
1534 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1535 intent.putExtra(
1536 EXTRA_LAST_ACTION,
1537 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1538 setIntent(intent);
1539 }
1540
1541 private static boolean emptyReference(final WeakReference<?> weakReference) {
1542 return weakReference == null || weakReference.get() == null;
1543 }
1544}