1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.PackageManager;
14import android.os.Build;
15import android.os.Bundle;
16import android.os.Handler;
17import android.os.PowerManager;
18import android.os.SystemClock;
19import android.util.Log;
20import android.util.Rational;
21import android.view.KeyEvent;
22import android.view.Menu;
23import android.view.MenuItem;
24import android.view.View;
25import android.view.WindowManager;
26import android.widget.Toast;
27
28import androidx.annotation.NonNull;
29import androidx.annotation.Nullable;
30import androidx.annotation.RequiresApi;
31import androidx.annotation.StringRes;
32import androidx.databinding.DataBindingUtil;
33
34import com.google.common.base.Optional;
35import com.google.common.base.Preconditions;
36import com.google.common.base.Throwables;
37import com.google.common.collect.ImmutableList;
38import com.google.common.collect.ImmutableSet;
39import com.google.common.util.concurrent.FutureCallback;
40import com.google.common.util.concurrent.Futures;
41
42import org.jetbrains.annotations.NotNull;
43import org.webrtc.RendererCommon;
44import org.webrtc.SurfaceViewRenderer;
45import org.webrtc.VideoTrack;
46
47import java.lang.ref.WeakReference;
48import java.util.Arrays;
49import java.util.Collections;
50import java.util.List;
51import java.util.Set;
52
53import eu.siacs.conversations.Config;
54import eu.siacs.conversations.R;
55import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.services.XmppConnectionService;
61import eu.siacs.conversations.ui.widget.DialpadView;
62import eu.siacs.conversations.ui.util.AvatarWorkerTask;
63import eu.siacs.conversations.ui.util.MainThreadExecutor;
64import eu.siacs.conversations.ui.util.Rationals;
65import eu.siacs.conversations.utils.PermissionUtils;
66import eu.siacs.conversations.utils.TimeFrameUtils;
67import eu.siacs.conversations.xml.Namespace;
68import eu.siacs.conversations.xmpp.Jid;
69import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
70import eu.siacs.conversations.xmpp.jingle.ContentAddition;
71import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
72import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
73import eu.siacs.conversations.xmpp.jingle.Media;
74import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
75
76public class RtpSessionActivity extends XmppActivity
77 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
78 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
79
80 public static final String EXTRA_WITH = "with";
81 public static final String EXTRA_SESSION_ID = "session_id";
82 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
83 public static final String EXTRA_LAST_ACTION = "last_action";
84 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
85 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
86 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
87
88 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
89
90 public static final List<RtpEndUserState> END_CARD =
91 Arrays.asList(
92 RtpEndUserState.APPLICATION_ERROR,
93 RtpEndUserState.SECURITY_ERROR,
94 RtpEndUserState.DECLINED_OR_BUSY,
95 RtpEndUserState.CONNECTIVITY_ERROR,
96 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
97 RtpEndUserState.RETRACTED);
98 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
99 Arrays.asList(
100 RtpEndUserState.APPLICATION_ERROR,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.SECURITY_ERROR);
103 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
104 Arrays.asList(
105 RtpEndUserState.CONNECTING,
106 RtpEndUserState.CONNECTED,
107 RtpEndUserState.RECONNECTING,
108 RtpEndUserState.INCOMING_CONTENT_ADD);
109 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
110 Arrays.asList(
111 RtpEndUserState.CONNECTED,
112 RtpEndUserState.RECONNECTING);
113 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
114 Arrays.asList(
115 RtpEndUserState.ACCEPTING_CALL,
116 RtpEndUserState.CONNECTING,
117 RtpEndUserState.RECONNECTING);
118 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
119 private static final int REQUEST_ACCEPT_CALL = 0x1111;
120 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
121 private static final int REQUEST_ADD_CONTENT = 0x1113;
122 private WeakReference<JingleRtpConnection> rtpConnectionReference;
123
124 private ActivityRtpSessionBinding binding;
125 private PowerManager.WakeLock mProximityWakeLock;
126
127 private final Handler mHandler = new Handler();
128 private final Runnable mTickExecutor =
129 new Runnable() {
130 @Override
131 public void run() {
132 updateCallDuration();
133 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
134 }
135 };
136
137 private static Set<Media> actionToMedia(final String action) {
138 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
139 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
140 } else {
141 return ImmutableSet.of(Media.AUDIO);
142 }
143 }
144
145 private static void addSink(
146 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
147 try {
148 videoTrack.addSink(surfaceViewRenderer);
149 } catch (final IllegalStateException e) {
150 Log.e(
151 Config.LOGTAG,
152 "possible race condition on trying to display video track. ignoring",
153 e);
154 }
155 }
156
157 @Override
158 public void onCreate(Bundle savedInstanceState) {
159 super.onCreate(savedInstanceState);
160 getWindow()
161 .addFlags(
162 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
163 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
164 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
165 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
166 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
167 setSupportActionBar(binding.toolbar);
168
169 binding.dialpad.setClickConsumer(tag -> {
170 final JingleRtpConnection connection =
171 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
172 if (connection != null) connection.applyDtmfTone(tag);
173 });
174
175 if (savedInstanceState != null) {
176 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
177 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
178 }
179 }
180
181 @Override
182 public boolean onCreateOptionsMenu(final Menu menu) {
183 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
184 final MenuItem help = menu.findItem(R.id.action_help);
185 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
186 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
187 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
188 help.setVisible(Config.HELP != null && isHelpButtonVisible());
189 gotoChat.setVisible(isSwitchToConversationVisible());
190 switchToVideo.setVisible(isSwitchToVideoVisible());
191 dialpad.setVisible(isAudioOnlyConversation());
192 return super.onCreateOptionsMenu(menu);
193 }
194
195 @Override
196 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
197 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
198 if (xmppConnectionService != null) {
199 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
200 return true;
201 }
202 }
203 }
204 return super.onKeyDown(keyCode, event);
205 }
206
207 private boolean isHelpButtonVisible() {
208 try {
209 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
210 } catch (IllegalStateException e) {
211 final Intent intent = getIntent();
212 final String state =
213 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
214 if (state != null) {
215 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
216 } else {
217 return false;
218 }
219 }
220 }
221
222 private boolean isSwitchToConversationVisible() {
223 final JingleRtpConnection connection =
224 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
225 return connection != null
226 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
227 }
228
229 private boolean isAudioOnlyConversation() {
230 final JingleRtpConnection connection =
231 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
232
233 return connection != null && !connection.getMedia().contains(Media.VIDEO);
234 }
235
236 private boolean isSwitchToVideoVisible() {
237 final JingleRtpConnection connection =
238 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
239 if (connection == null) {
240 return false;
241 }
242 return connection.isSwitchToVideoAvailable();
243 }
244
245 private void switchToConversation() {
246 final Contact contact = getWith();
247 final Conversation conversation =
248 xmppConnectionService.findOrCreateConversation(
249 contact.getAccount(), contact.getJid(), false, true);
250 switchToConversation(conversation);
251 }
252
253 private void toggleDialpadVisibility() {
254 if (binding.dialpad.getVisibility() == View.VISIBLE) {
255 binding.dialpad.setVisibility(View.GONE);
256 }
257 else {
258 binding.dialpad.setVisibility(View.VISIBLE);
259 }
260 }
261
262 public boolean onOptionsItemSelected(final MenuItem item) {
263 switch (item.getItemId()) {
264 case R.id.action_help:
265 launchHelpInBrowser();
266 return true;
267 case R.id.action_goto_chat:
268 switchToConversation();
269 return true;
270 case R.id.action_dialpad:
271 toggleDialpadVisibility();
272 return true;
273 case R.id.action_switch_to_video:
274 requestPermissionAndSwitchToVideo();
275 return true;
276 }
277 return super.onOptionsItemSelected(item);
278 }
279
280 private void launchHelpInBrowser() {
281 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
282 try {
283 startActivity(intent);
284 } catch (final ActivityNotFoundException e) {
285 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
286 .show();
287 }
288 }
289
290 private void endCall(View view) {
291 endCall();
292 }
293
294 private void endCall() {
295 if (this.rtpConnectionReference == null) {
296 retractSessionProposal();
297 finish();
298 } else {
299 requireRtpConnection().endCall();
300 }
301 }
302
303 private void retractSessionProposal() {
304 final Intent intent = getIntent();
305 final String action = intent.getAction();
306 final Account account = extractAccount(intent);
307 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
308 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
309 if (!Intent.ACTION_VIEW.equals(action)
310 || state == null
311 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
312 resetIntent(
313 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
314 }
315 xmppConnectionService
316 .getJingleConnectionManager()
317 .retractSessionProposal(account, with.asBareJid());
318 }
319
320 private void rejectCall(View view) {
321 requireRtpConnection().rejectCall();
322 finish();
323 }
324
325 private void acceptCall(View view) {
326 requestPermissionsAndAcceptCall();
327 }
328
329 private void acceptContentAdd() {
330 try {
331 requireRtpConnection()
332 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
333 } catch (final IllegalStateException e) {
334 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
335 }
336 }
337
338 private void requestPermissionAndSwitchToVideo() {
339 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
340 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
341 switchToVideo();
342 }
343 }
344
345 private void switchToVideo() {
346 try {
347 requireRtpConnection().addMedia(Media.VIDEO);
348 } catch (final IllegalStateException e) {
349 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
350 }
351 }
352
353 private void acceptContentAdd(final ContentAddition contentAddition) {
354 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
355 Log.d(Config.LOGTAG,"ignore press on content-accept button");
356 return;
357 }
358 requestPermissionAndAcceptContentAdd(contentAddition);
359 }
360
361 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
362 final List<String> permissions = permissions(contentAddition.media());
363 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
364 requireRtpConnection().acceptContentAdd(contentAddition.summary);
365 }
366 }
367
368 private void rejectContentAdd(final View view) {
369 requireRtpConnection().rejectContentAdd();
370 }
371
372 private void requestPermissionsAndAcceptCall() {
373 final List<String> permissions = permissions(getMedia());
374 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
375 putScreenInCallMode();
376 checkRecorderAndAcceptCall();
377 }
378 }
379
380 private List<String> permissions(final Set<Media> media) {
381 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
382 if (media.contains(Media.VIDEO)) {
383 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
384 } else {
385 permissions.add(Manifest.permission.RECORD_AUDIO);
386 }
387 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
388 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
389 }
390 return permissions.build();
391 }
392
393 private void checkRecorderAndAcceptCall() {
394 checkMicrophoneAvailabilityAsync();
395 try {
396 requireRtpConnection().acceptCall();
397 } catch (final IllegalStateException e) {
398 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
399 }
400 }
401
402 private void checkMicrophoneAvailabilityAsync() {
403 new Thread(new MicrophoneAvailabilityCheck(this)).start();
404 }
405
406 private static class MicrophoneAvailabilityCheck implements Runnable {
407
408 private final WeakReference<Activity> activityReference;
409
410 private MicrophoneAvailabilityCheck(final Activity activity) {
411 this.activityReference = new WeakReference<>(activity);
412 }
413
414 @Override
415 public void run() {
416 final long start = SystemClock.elapsedRealtime();
417 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
418 final long stop = SystemClock.elapsedRealtime();
419 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
420 if (isMicrophoneAvailable) {
421 return;
422 }
423 final Activity activity = activityReference.get();
424 if (activity == null) {
425 return;
426 }
427 activity.runOnUiThread(
428 () ->
429 Toast.makeText(
430 activity,
431 R.string.microphone_unavailable,
432 Toast.LENGTH_LONG)
433 .show());
434 }
435 }
436
437 private void putScreenInCallMode() {
438 putScreenInCallMode(requireRtpConnection().getMedia());
439 }
440
441 private void putScreenInCallMode(final Set<Media> media) {
442 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
443 if (!media.contains(Media.VIDEO)) {
444 final JingleRtpConnection rtpConnection =
445 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
446 final AppRTCAudioManager audioManager =
447 rtpConnection == null ? null : rtpConnection.getAudioManager();
448 if (audioManager == null
449 || audioManager.getSelectedAudioDevice()
450 == AppRTCAudioManager.AudioDevice.EARPIECE) {
451 acquireProximityWakeLock();
452 }
453 }
454 }
455
456 @SuppressLint("WakelockTimeout")
457 private void acquireProximityWakeLock() {
458 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
459 if (powerManager == null) {
460 Log.e(Config.LOGTAG, "power manager not available");
461 return;
462 }
463 if (isFinishing()) {
464 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
465 return;
466 }
467 if (this.mProximityWakeLock == null) {
468 this.mProximityWakeLock =
469 powerManager.newWakeLock(
470 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
471 }
472 if (!this.mProximityWakeLock.isHeld()) {
473 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
474 this.mProximityWakeLock.acquire();
475 }
476 }
477
478 private void releaseProximityWakeLock() {
479 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
480 Log.d(Config.LOGTAG, "releasing proximity wake lock");
481 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
482 this.mProximityWakeLock = null;
483 }
484 }
485
486 private void putProximityWakeLockInProperState(
487 final AppRTCAudioManager.AudioDevice audioDevice) {
488 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
489 acquireProximityWakeLock();
490 } else {
491 releaseProximityWakeLock();
492 }
493 }
494
495 @Override
496 protected void refreshUiReal() {}
497
498 @Override
499 public void onNewIntent(final Intent intent) {
500 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
501 super.onNewIntent(intent);
502 setIntent(intent);
503 if (xmppConnectionService == null) {
504 Log.d(
505 Config.LOGTAG,
506 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
507 return;
508 }
509 final Account account = extractAccount(intent);
510 final String action = intent.getAction();
511 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
512 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
513 if (sessionId != null) {
514 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
515 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
516 return;
517 }
518 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
519 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
520 requestPermissionsAndAcceptCall();
521 resetIntent(intent.getExtras());
522 }
523 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
524 proposeJingleRtpSession(account, with, actionToMedia(action));
525 setWith(account.getRoster().getContact(with), null);
526 } else {
527 throw new IllegalStateException("received onNewIntent without sessionId");
528 }
529 }
530
531 @Override
532 void onBackendConnected() {
533 final Intent intent = getIntent();
534 final String action = intent.getAction();
535 final Account account = extractAccount(intent);
536 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
537 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
538 if (sessionId != null) {
539 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
540 return;
541 }
542 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
543 Log.d(Config.LOGTAG, "intent action was accept");
544 requestPermissionsAndAcceptCall();
545 resetIntent(intent.getExtras());
546 }
547 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
548 proposeJingleRtpSession(account, with, actionToMedia(action));
549 setWith(account.getRoster().getContact(with), null);
550 } else if (Intent.ACTION_VIEW.equals(action)) {
551 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
552 final RtpEndUserState state =
553 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
554 if (state != null) {
555 Log.d(Config.LOGTAG, "restored last state from intent extra");
556 updateButtonConfiguration(state);
557 updateVerifiedShield(false);
558 updateStateDisplay(state);
559 updateIncomingCallScreen(state);
560 invalidateOptionsMenu();
561 }
562 setWith(account.getRoster().getContact(with), state);
563 if (xmppConnectionService
564 .getJingleConnectionManager()
565 .fireJingleRtpConnectionStateUpdates()) {
566 return;
567 }
568 if (END_CARD.contains(state)
569 || xmppConnectionService
570 .getJingleConnectionManager()
571 .hasMatchingProposal(account, with)) {
572 return;
573 }
574 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
575 finish();
576 }
577 }
578
579 private void setWith(final RtpEndUserState state) {
580 setWith(getWith(), state);
581 }
582
583 private void setWith(final Contact contact, final RtpEndUserState state) {
584 binding.with.setText(contact.getDisplayName());
585 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
586 .contains(state)) {
587 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
588 binding.withJid.setVisibility(View.VISIBLE);
589 } else {
590 binding.withJid.setVisibility(View.GONE);
591 }
592 }
593
594 private void proposeJingleRtpSession(
595 final Account account, final Jid with, final Set<Media> media) {
596 checkMicrophoneAvailabilityAsync();
597 if (with.isBareJid()) {
598 xmppConnectionService
599 .getJingleConnectionManager()
600 .proposeJingleRtpSession(account, with, media);
601 } else {
602 final String sessionId =
603 xmppConnectionService
604 .getJingleConnectionManager()
605 .initializeRtpSession(account, with, media);
606 initializeActivityWithRunningRtpSession(account, with, sessionId);
607 resetIntent(account, with, sessionId);
608 }
609 putScreenInCallMode(media);
610 }
611
612 @Override
613 public void onRequestPermissionsResult(
614 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
615 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
616 final PermissionUtils.PermissionResult permissionResult =
617 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
618 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
619 if (requestCode == REQUEST_ACCEPT_CALL) {
620 checkRecorderAndAcceptCall();
621 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
622 acceptContentAdd();
623 } else if (requestCode == REQUEST_ADD_CONTENT) {
624 switchToVideo();
625 }
626 } else {
627 @StringRes int res;
628 final String firstDenied =
629 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
630 if (firstDenied == null) {
631 return;
632 }
633 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
634 res = R.string.no_microphone_permission;
635 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
636 res = R.string.no_camera_permission;
637 } else {
638 throw new IllegalStateException("Invalid permission result request");
639 }
640 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
641 .show();
642 }
643 }
644
645 @Override
646 public void onStart() {
647 super.onStart();
648 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
649 this.binding.remoteVideo.setOnAspectRatioChanged(this);
650 }
651
652 @Override
653 public void onStop() {
654 mHandler.removeCallbacks(mTickExecutor);
655 binding.remoteVideo.release();
656 binding.remoteVideo.setOnAspectRatioChanged(null);
657 binding.localVideo.release();
658 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
659 final JingleRtpConnection jingleRtpConnection =
660 weakReference == null ? null : weakReference.get();
661 if (jingleRtpConnection != null) {
662 releaseVideoTracks(jingleRtpConnection);
663 }
664 releaseProximityWakeLock();
665 super.onStop();
666 }
667
668 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
669 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
670 if (remoteVideo.isPresent()) {
671 remoteVideo.get().removeSink(binding.remoteVideo);
672 }
673 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
674 if (localVideo.isPresent()) {
675 localVideo.get().removeSink(binding.localVideo);
676 }
677 }
678
679 @Override
680 public void onBackPressed() {
681 if (isConnected()) {
682 if (switchToPictureInPicture()) {
683 return;
684 }
685 } else {
686 endCall();
687 }
688 super.onBackPressed();
689 }
690
691 @Override
692 public void onUserLeaveHint() {
693 super.onUserLeaveHint();
694 if (switchToPictureInPicture()) {
695 return;
696 }
697 // TODO apparently this method is not getting called on Android 10 when using the task
698 // switcher
699 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
700 retractSessionProposal();
701 }
702 }
703
704 private boolean isConnected() {
705 final JingleRtpConnection connection =
706 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
707 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
708 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
709 }
710
711 private boolean switchToPictureInPicture() {
712 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
713 if (shouldBePictureInPicture()) {
714 startPictureInPicture();
715 return true;
716 }
717 }
718 return false;
719 }
720
721 @RequiresApi(api = Build.VERSION_CODES.O)
722 private void startPictureInPicture() {
723 try {
724 final Rational rational = this.binding.remoteVideo.getAspectRatio();
725 final Rational clippedRational = Rationals.clip(rational);
726 Log.d(
727 Config.LOGTAG,
728 "suggested rational " + rational + ". clipped to " + clippedRational);
729 enterPictureInPictureMode(
730 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
731 } catch (final IllegalStateException e) {
732 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
733 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
734 }
735 }
736
737 @Override
738 public void onAspectRatioChanged(final Rational rational) {
739 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
740 final Rational clippedRational = Rationals.clip(rational);
741 Log.d(
742 Config.LOGTAG,
743 "suggested rational after aspect ratio change "
744 + rational
745 + ". clipped to "
746 + clippedRational);
747 setPictureInPictureParams(
748 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
749 }
750 }
751
752 private boolean deviceSupportsPictureInPicture() {
753 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
754 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
755 } else {
756 return false;
757 }
758 }
759
760 private boolean shouldBePictureInPicture() {
761 try {
762 final JingleRtpConnection rtpConnection = requireRtpConnection();
763 return rtpConnection.getMedia().contains(Media.VIDEO)
764 && Arrays.asList(
765 RtpEndUserState.ACCEPTING_CALL,
766 RtpEndUserState.CONNECTING,
767 RtpEndUserState.CONNECTED)
768 .contains(rtpConnection.getEndUserState());
769 } catch (final IllegalStateException e) {
770 return false;
771 }
772 }
773
774 private boolean initializeActivityWithRunningRtpSession(
775 final Account account, Jid with, String sessionId) {
776 final WeakReference<JingleRtpConnection> reference =
777 xmppConnectionService
778 .getJingleConnectionManager()
779 .findJingleRtpConnection(account, with, sessionId);
780 if (reference == null || reference.get() == null) {
781 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
782 xmppConnectionService
783 .getJingleConnectionManager()
784 .getTerminalSessionState(with, sessionId);
785 if (terminatedRtpSession == null) {
786 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
787 finish();
788 return true;
789 }
790 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
791 return true;
792 }
793 this.rtpConnectionReference = reference;
794 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
795 final boolean verified = requireRtpConnection().isVerified();
796 if (currentState == RtpEndUserState.ENDED) {
797 finish();
798 return true;
799 }
800 final Set<Media> media = getMedia();
801 final ContentAddition contentAddition = getPendingContentAddition();
802 if (currentState == RtpEndUserState.INCOMING_CALL) {
803 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
804 }
805 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
806 requireRtpConnection().getState())) {
807 putScreenInCallMode();
808 }
809 setWith(currentState);
810 updateVideoViews(currentState);
811 updateStateDisplay(currentState, media, contentAddition);
812 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
813 updateButtonConfiguration(currentState, media, contentAddition);
814 updateIncomingCallScreen(currentState);
815 invalidateOptionsMenu();
816 return false;
817 }
818
819 private void initializeWithTerminatedSessionState(
820 final Account account,
821 final Jid with,
822 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
823 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
824 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
825 finish();
826 return;
827 }
828 final RtpEndUserState state = terminatedRtpSession.state;
829 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
830 updateButtonConfiguration(state);
831 updateStateDisplay(state);
832 updateIncomingCallScreen(state);
833 updateCallDuration();
834 updateVerifiedShield(false);
835 invalidateOptionsMenu();
836 setWith(account.getRoster().getContact(with), state);
837 }
838
839 private void reInitializeActivityWithRunningRtpSession(
840 final Account account, Jid with, String sessionId) {
841 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
842 resetIntent(account, with, sessionId);
843 }
844
845 private void resetIntent(final Account account, final Jid with, final String sessionId) {
846 final Intent intent = new Intent(Intent.ACTION_VIEW);
847 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
848 intent.putExtra(EXTRA_WITH, with.toEscapedString());
849 intent.putExtra(EXTRA_SESSION_ID, sessionId);
850 setIntent(intent);
851 }
852
853 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
854 surfaceViewRenderer.setVisibility(View.VISIBLE);
855 try {
856 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
857 } catch (final IllegalStateException e) {
858 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
859 }
860 surfaceViewRenderer.setEnableHardwareScaler(true);
861 }
862
863 private void updateStateDisplay(final RtpEndUserState state) {
864 updateStateDisplay(state, Collections.emptySet(), null);
865 }
866
867 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
868 switch (state) {
869 case INCOMING_CALL:
870 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
871 if (media.contains(Media.VIDEO)) {
872 setTitle(R.string.rtp_state_incoming_video_call);
873 } else {
874 setTitle(R.string.rtp_state_incoming_call);
875 }
876 break;
877 case INCOMING_CONTENT_ADD:
878 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
879 setTitle(R.string.rtp_state_content_add_video);
880 } else {
881 setTitle(R.string.rtp_state_content_add);
882 }
883 break;
884 case CONNECTING:
885 setTitle(R.string.rtp_state_connecting);
886 break;
887 case CONNECTED:
888 setTitle(R.string.rtp_state_connected);
889 break;
890 case RECONNECTING:
891 setTitle(R.string.rtp_state_reconnecting);
892 break;
893 case ACCEPTING_CALL:
894 setTitle(R.string.rtp_state_accepting_call);
895 break;
896 case ENDING_CALL:
897 setTitle(R.string.rtp_state_ending_call);
898 break;
899 case FINDING_DEVICE:
900 setTitle(R.string.rtp_state_finding_device);
901 break;
902 case RINGING:
903 setTitle(R.string.rtp_state_ringing);
904 break;
905 case DECLINED_OR_BUSY:
906 setTitle(R.string.rtp_state_declined_or_busy);
907 break;
908 case CONNECTIVITY_ERROR:
909 setTitle(R.string.rtp_state_connectivity_error);
910 break;
911 case CONNECTIVITY_LOST_ERROR:
912 setTitle(R.string.rtp_state_connectivity_lost_error);
913 break;
914 case RETRACTED:
915 setTitle(R.string.rtp_state_retracted);
916 break;
917 case APPLICATION_ERROR:
918 setTitle(R.string.rtp_state_application_failure);
919 break;
920 case SECURITY_ERROR:
921 setTitle(R.string.rtp_state_security_error);
922 break;
923 case ENDED:
924 throw new IllegalStateException(
925 "Activity should have called finishAndReleaseWakeLock();");
926 default:
927 throw new IllegalStateException(
928 String.format("State %s has not been handled in UI", state));
929 }
930 }
931
932 private void updateVerifiedShield(final boolean verified) {
933 if (isPictureInPicture()) {
934 this.binding.verified.setVisibility(View.GONE);
935 return;
936 }
937 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
938 }
939
940 private void updateIncomingCallScreen(final RtpEndUserState state) {
941 updateIncomingCallScreen(state, null);
942 }
943
944 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
945 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
946 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
947 if (show) {
948 binding.contactPhoto.setVisibility(View.VISIBLE);
949 if (contact == null) {
950 AvatarWorkerTask.loadAvatar(
951 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
952 } else {
953 AvatarWorkerTask.loadAvatar(
954 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
955 }
956 } else {
957 binding.contactPhoto.setVisibility(View.GONE);
958 }
959 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
960 binding.usingAccount.setVisibility(View.VISIBLE);
961 binding.usingAccount.setText(
962 getString(
963 R.string.using_account,
964 account.getJid().asBareJid().toEscapedString()));
965 } else {
966 binding.usingAccount.setVisibility(View.GONE);
967 binding.contactPhoto.setVisibility(View.GONE);
968 }
969 }
970
971 private Set<Media> getMedia() {
972 return requireRtpConnection().getMedia();
973 }
974
975 public ContentAddition getPendingContentAddition() {
976 return requireRtpConnection().getPendingContentAddition();
977 }
978
979 private void updateButtonConfiguration(final RtpEndUserState state) {
980 updateButtonConfiguration(state, Collections.emptySet(), null);
981 }
982
983 @SuppressLint("RestrictedApi")
984 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
985 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
986 this.binding.rejectCall.setVisibility(View.INVISIBLE);
987 this.binding.endCall.setVisibility(View.INVISIBLE);
988 this.binding.acceptCall.setVisibility(View.INVISIBLE);
989 } else if (state == RtpEndUserState.INCOMING_CALL) {
990 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
991 this.binding.rejectCall.setOnClickListener(this::rejectCall);
992 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
993 this.binding.rejectCall.setVisibility(View.VISIBLE);
994 this.binding.endCall.setVisibility(View.INVISIBLE);
995 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
996 this.binding.acceptCall.setOnClickListener(this::acceptCall);
997 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
998 this.binding.acceptCall.setVisibility(View.VISIBLE);
999 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
1000 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
1001 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
1002 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1003 this.binding.rejectCall.setVisibility(View.VISIBLE);
1004 this.binding.endCall.setVisibility(View.INVISIBLE);
1005 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1006 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1007 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
1008 this.binding.acceptCall.setVisibility(View.VISIBLE);
1009 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
1010 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1011 this.binding.rejectCall.setOnClickListener(this::exit);
1012 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1013 this.binding.rejectCall.setVisibility(View.VISIBLE);
1014 this.binding.endCall.setVisibility(View.INVISIBLE);
1015 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1016 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1017 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1018 this.binding.acceptCall.setVisibility(View.VISIBLE);
1019 } else if (asList(
1020 RtpEndUserState.CONNECTIVITY_ERROR,
1021 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1022 RtpEndUserState.APPLICATION_ERROR,
1023 RtpEndUserState.RETRACTED,
1024 RtpEndUserState.SECURITY_ERROR)
1025 .contains(state)) {
1026 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1027 this.binding.rejectCall.setOnClickListener(this::exit);
1028 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1029 this.binding.rejectCall.setVisibility(View.VISIBLE);
1030 this.binding.endCall.setVisibility(View.INVISIBLE);
1031 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1032 this.binding.acceptCall.setOnClickListener(this::retry);
1033 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1034 this.binding.acceptCall.setVisibility(View.VISIBLE);
1035 } else {
1036 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1037 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1038 this.binding.endCall.setOnClickListener(this::endCall);
1039 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1040 this.binding.endCall.setVisibility(View.VISIBLE);
1041 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1042 }
1043 updateInCallButtonConfiguration(state, media);
1044 }
1045
1046 private boolean isPictureInPicture() {
1047 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1048 return isInPictureInPictureMode();
1049 } else {
1050 return false;
1051 }
1052 }
1053
1054 private void updateInCallButtonConfiguration() {
1055 updateInCallButtonConfiguration(
1056 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1057 }
1058
1059 @SuppressLint("RestrictedApi")
1060 private void updateInCallButtonConfiguration(
1061 final RtpEndUserState state, final Set<Media> media) {
1062 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1063 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1064 if (media.contains(Media.VIDEO)) {
1065 final JingleRtpConnection rtpConnection = requireRtpConnection();
1066 updateInCallButtonConfigurationVideo(
1067 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1068 } else {
1069 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1070 updateInCallButtonConfigurationSpeaker(
1071 audioManager.getSelectedAudioDevice(),
1072 audioManager.getAudioDevices().size());
1073 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1074 }
1075 if (media.contains(Media.AUDIO)) {
1076 updateInCallButtonConfigurationMicrophone(
1077 requireRtpConnection().isMicrophoneEnabled());
1078 } else {
1079 this.binding.inCallActionLeft.setVisibility(View.GONE);
1080 }
1081 } else {
1082 this.binding.inCallActionLeft.setVisibility(View.GONE);
1083 this.binding.inCallActionRight.setVisibility(View.GONE);
1084 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1085 }
1086 }
1087
1088 @SuppressLint("RestrictedApi")
1089 private void updateInCallButtonConfigurationSpeaker(
1090 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1091 switch (selectedAudioDevice) {
1092 case EARPIECE:
1093 this.binding.inCallActionRight.setImageResource(
1094 R.drawable.ic_volume_off_black_24dp);
1095 if (numberOfChoices >= 2) {
1096 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1097 } else {
1098 this.binding.inCallActionRight.setOnClickListener(null);
1099 this.binding.inCallActionRight.setClickable(false);
1100 }
1101 break;
1102 case WIRED_HEADSET:
1103 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1104 this.binding.inCallActionRight.setOnClickListener(null);
1105 this.binding.inCallActionRight.setClickable(false);
1106 break;
1107 case SPEAKER_PHONE:
1108 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1109 if (numberOfChoices >= 2) {
1110 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1111 } else {
1112 this.binding.inCallActionRight.setOnClickListener(null);
1113 this.binding.inCallActionRight.setClickable(false);
1114 }
1115 break;
1116 case BLUETOOTH:
1117 this.binding.inCallActionRight.setImageResource(
1118 R.drawable.ic_bluetooth_audio_black_24dp);
1119 this.binding.inCallActionRight.setOnClickListener(null);
1120 this.binding.inCallActionRight.setClickable(false);
1121 break;
1122 }
1123 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1124 }
1125
1126 @SuppressLint("RestrictedApi")
1127 private void updateInCallButtonConfigurationVideo(
1128 final boolean videoEnabled, final boolean isCameraSwitchable) {
1129 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1130 if (isCameraSwitchable) {
1131 this.binding.inCallActionFarRight.setImageResource(
1132 R.drawable.ic_flip_camera_android_black_24dp);
1133 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1134 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1135 } else {
1136 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1137 }
1138 if (videoEnabled) {
1139 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1140 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1141 } else {
1142 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1143 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1144 }
1145 }
1146
1147 private void switchCamera(final View view) {
1148 Futures.addCallback(
1149 requireRtpConnection().switchCamera(),
1150 new FutureCallback<Boolean>() {
1151 @Override
1152 public void onSuccess(@Nullable Boolean isFrontCamera) {
1153 binding.localVideo.setMirror(isFrontCamera);
1154 }
1155
1156 @Override
1157 public void onFailure(@NonNull final Throwable throwable) {
1158 Log.d(
1159 Config.LOGTAG,
1160 "could not switch camera",
1161 Throwables.getRootCause(throwable));
1162 Toast.makeText(
1163 RtpSessionActivity.this,
1164 R.string.could_not_switch_camera,
1165 Toast.LENGTH_LONG)
1166 .show();
1167 }
1168 },
1169 MainThreadExecutor.getInstance());
1170 }
1171
1172 private void enableVideo(View view) {
1173 try {
1174 requireRtpConnection().setVideoEnabled(true);
1175 } catch (final IllegalStateException e) {
1176 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1177 return;
1178 }
1179 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1180 }
1181
1182 private void disableVideo(View view) {
1183 final JingleRtpConnection rtpConnection = requireRtpConnection();
1184 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1185 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1186 rtpConnection.retractContentAdd();
1187 return;
1188 }
1189 requireRtpConnection().setVideoEnabled(false);
1190 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1191 }
1192
1193 @SuppressLint("RestrictedApi")
1194 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1195 if (microphoneEnabled) {
1196 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1197 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1198 } else {
1199 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1200 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1201 }
1202 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1203 }
1204
1205 private void updateCallDuration() {
1206 final JingleRtpConnection connection =
1207 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1208 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1209 this.binding.duration.setVisibility(View.GONE);
1210 return;
1211 }
1212 if (connection.zeroDuration()) {
1213 this.binding.duration.setVisibility(View.GONE);
1214 } else {
1215 this.binding.duration.setText(
1216 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1217 this.binding.duration.setVisibility(View.VISIBLE);
1218 }
1219 }
1220
1221 private void updateVideoViews(final RtpEndUserState state) {
1222 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1223 binding.localVideo.setVisibility(View.GONE);
1224 binding.localVideo.release();
1225 binding.remoteVideoWrapper.setVisibility(View.GONE);
1226 binding.remoteVideo.release();
1227 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1228 if (isPictureInPicture()) {
1229 binding.appBarLayout.setVisibility(View.GONE);
1230 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1231 if (Arrays.asList(
1232 RtpEndUserState.APPLICATION_ERROR,
1233 RtpEndUserState.CONNECTIVITY_ERROR,
1234 RtpEndUserState.SECURITY_ERROR)
1235 .contains(state)) {
1236 binding.pipWarning.setVisibility(View.VISIBLE);
1237 binding.pipWaiting.setVisibility(View.GONE);
1238 } else {
1239 binding.pipWarning.setVisibility(View.GONE);
1240 binding.pipWaiting.setVisibility(View.GONE);
1241 }
1242 } else {
1243 binding.appBarLayout.setVisibility(View.VISIBLE);
1244 binding.pipPlaceholder.setVisibility(View.GONE);
1245 }
1246 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1247 return;
1248 }
1249 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1250 binding.localVideo.setVisibility(View.GONE);
1251 binding.remoteVideoWrapper.setVisibility(View.GONE);
1252 binding.appBarLayout.setVisibility(View.GONE);
1253 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1254 binding.pipWarning.setVisibility(View.GONE);
1255 binding.pipWaiting.setVisibility(View.VISIBLE);
1256 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1257 return;
1258 }
1259 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1260 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1261 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1262 // paint local view over remote view
1263 binding.localVideo.setZOrderMediaOverlay(true);
1264 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1265 addSink(localVideoTrack.get(), binding.localVideo);
1266 } else {
1267 binding.localVideo.setVisibility(View.GONE);
1268 }
1269 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1270 if (remoteVideoTrack.isPresent()) {
1271 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1272 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1273 binding.remoteVideo.setScalingType(
1274 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1275 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1276 if (state == RtpEndUserState.CONNECTED) {
1277 binding.appBarLayout.setVisibility(View.GONE);
1278 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1279 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1280 } else {
1281 binding.appBarLayout.setVisibility(View.VISIBLE);
1282 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1283 binding.remoteVideoWrapper.setVisibility(View.GONE);
1284 }
1285 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1286 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1287 } else {
1288 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1289 }
1290 } else {
1291 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1292 binding.remoteVideoWrapper.setVisibility(View.GONE);
1293 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1294 }
1295 }
1296
1297 private Optional<VideoTrack> getLocalVideoTrack() {
1298 final JingleRtpConnection connection =
1299 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1300 if (connection == null) {
1301 return Optional.absent();
1302 }
1303 return connection.getLocalVideoTrack();
1304 }
1305
1306 private Optional<VideoTrack> getRemoteVideoTrack() {
1307 final JingleRtpConnection connection =
1308 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1309 if (connection == null) {
1310 return Optional.absent();
1311 }
1312 return connection.getRemoteVideoTrack();
1313 }
1314
1315 private void disableMicrophone(View view) {
1316 final JingleRtpConnection rtpConnection = requireRtpConnection();
1317 if (rtpConnection.setMicrophoneEnabled(false)) {
1318 updateInCallButtonConfiguration();
1319 }
1320 }
1321
1322 private void enableMicrophone(View view) {
1323 final JingleRtpConnection rtpConnection = requireRtpConnection();
1324 if (rtpConnection.setMicrophoneEnabled(true)) {
1325 updateInCallButtonConfiguration();
1326 }
1327 }
1328
1329 private void switchToEarpiece(View view) {
1330 requireRtpConnection()
1331 .getAudioManager()
1332 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1333 acquireProximityWakeLock();
1334 }
1335
1336 private void switchToSpeaker(View view) {
1337 requireRtpConnection()
1338 .getAudioManager()
1339 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1340 releaseProximityWakeLock();
1341 }
1342
1343 private void retry(View view) {
1344 final Intent intent = getIntent();
1345 final Account account = extractAccount(intent);
1346 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1347 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1348 final String action = intent.getAction();
1349 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1350 this.rtpConnectionReference = null;
1351 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1352 proposeJingleRtpSession(account, with, media);
1353 }
1354
1355 private void exit(final View view) {
1356 finish();
1357 }
1358
1359 private void recordVoiceMail(final View view) {
1360 final Intent intent = getIntent();
1361 final Account account = extractAccount(intent);
1362 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1363 final Conversation conversation =
1364 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1365 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1366 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1367 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1368 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1369 launchIntent.putExtra(
1370 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1371 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1372 startActivity(launchIntent);
1373 finish();
1374 }
1375
1376 private Contact getWith() {
1377 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1378 final Account account = id.account;
1379 return account.getRoster().getContact(id.with);
1380 }
1381
1382 private JingleRtpConnection requireRtpConnection() {
1383 final JingleRtpConnection connection =
1384 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1385 if (connection == null) {
1386 throw new IllegalStateException("No RTP connection found");
1387 }
1388 return connection;
1389 }
1390
1391 @Override
1392 public void onJingleRtpConnectionUpdate(
1393 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1394 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1395 if (END_CARD.contains(state)) {
1396 Log.d(Config.LOGTAG, "end card reached");
1397 releaseProximityWakeLock();
1398 runOnUiThread(
1399 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1400 }
1401 if (with.isBareJid()) {
1402 updateRtpSessionProposalState(account, with, state);
1403 return;
1404 }
1405 if (emptyReference(this.rtpConnectionReference)) {
1406 if (END_CARD.contains(state)) {
1407 Log.d(Config.LOGTAG, "not reinitializing session");
1408 return;
1409 }
1410 // this happens when going from proposed session to actual session
1411 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1412 return;
1413 }
1414 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1415 final boolean verified = requireRtpConnection().isVerified();
1416 final Set<Media> media = getMedia();
1417 final ContentAddition contentAddition = getPendingContentAddition();
1418 final Contact contact = getWith();
1419 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1420 if (state == RtpEndUserState.ENDED) {
1421 finish();
1422 return;
1423 }
1424 runOnUiThread(
1425 () -> {
1426 updateStateDisplay(state, media, contentAddition);
1427 updateVerifiedShield(
1428 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1429 updateButtonConfiguration(state, media, contentAddition);
1430 updateVideoViews(state);
1431 updateIncomingCallScreen(state, contact);
1432 invalidateOptionsMenu();
1433 });
1434 if (END_CARD.contains(state)) {
1435 final JingleRtpConnection rtpConnection = requireRtpConnection();
1436 resetIntent(account, with, state, rtpConnection.getMedia());
1437 releaseVideoTracks(rtpConnection);
1438 this.rtpConnectionReference = null;
1439 }
1440 } else {
1441 Log.d(Config.LOGTAG, "received update for other rtp session");
1442 }
1443 }
1444
1445 @Override
1446 public void onAudioDeviceChanged(
1447 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1448 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1449 Log.d(
1450 Config.LOGTAG,
1451 "onAudioDeviceChanged in activity: selected:"
1452 + selectedAudioDevice
1453 + ", available:"
1454 + availableAudioDevices);
1455 try {
1456 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1457 final Set<Media> media = getMedia();
1458 if (END_CARD.contains(endUserState)) {
1459 Log.d(
1460 Config.LOGTAG,
1461 "onAudioDeviceChanged() nothing to do because end card has been reached");
1462 } else {
1463 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1464 final AppRTCAudioManager audioManager =
1465 requireRtpConnection().getAudioManager();
1466 updateInCallButtonConfigurationSpeaker(
1467 audioManager.getSelectedAudioDevice(),
1468 audioManager.getAudioDevices().size());
1469 }
1470 Log.d(
1471 Config.LOGTAG,
1472 "put proximity wake lock into proper state after device update");
1473 putProximityWakeLockInProperState(selectedAudioDevice);
1474 }
1475 } catch (final IllegalStateException e) {
1476 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1477 }
1478 }
1479
1480 @Override
1481 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1482 super.onSaveInstanceState(outState);
1483 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1484 }
1485
1486 private void updateRtpSessionProposalState(
1487 final Account account, final Jid with, final RtpEndUserState state) {
1488 final Intent currentIntent = getIntent();
1489 final String withExtra =
1490 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1491 if (withExtra == null) {
1492 return;
1493 }
1494 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1495 runOnUiThread(
1496 () -> {
1497 updateVerifiedShield(false);
1498 updateStateDisplay(state);
1499 updateButtonConfiguration(state);
1500 updateIncomingCallScreen(state);
1501 invalidateOptionsMenu();
1502 });
1503 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1504 }
1505 }
1506
1507 private void resetIntent(final Bundle extras) {
1508 final Intent intent = new Intent(Intent.ACTION_VIEW);
1509 intent.putExtras(extras);
1510 setIntent(intent);
1511 }
1512
1513 private void resetIntent(
1514 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1515 final Intent intent = new Intent(Intent.ACTION_VIEW);
1516 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1517 if (account.getRoster()
1518 .getContact(with)
1519 .getPresences()
1520 .anySupport(Namespace.JINGLE_MESSAGE)) {
1521 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1522 } else {
1523 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1524 }
1525 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1526 intent.putExtra(
1527 EXTRA_LAST_ACTION,
1528 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1529 setIntent(intent);
1530 }
1531
1532 private static boolean emptyReference(final WeakReference<?> weakReference) {
1533 return weakReference == null || weakReference.get() == null;
1534 }
1535}