1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.PackageManager;
14import android.os.Build;
15import android.os.Bundle;
16import android.os.Handler;
17import android.os.PowerManager;
18import android.os.SystemClock;
19import android.util.Log;
20import android.util.Rational;
21import android.view.KeyEvent;
22import android.view.Menu;
23import android.view.MenuItem;
24import android.view.View;
25import android.view.WindowManager;
26import android.widget.Toast;
27
28import androidx.annotation.NonNull;
29import androidx.annotation.Nullable;
30import androidx.annotation.RequiresApi;
31import androidx.annotation.StringRes;
32import androidx.databinding.DataBindingUtil;
33
34import com.google.common.base.Optional;
35import com.google.common.base.Preconditions;
36import com.google.common.base.Throwables;
37import com.google.common.collect.ImmutableList;
38import com.google.common.collect.ImmutableSet;
39import com.google.common.util.concurrent.FutureCallback;
40import com.google.common.util.concurrent.Futures;
41
42import org.jetbrains.annotations.NotNull;
43import org.webrtc.RendererCommon;
44import org.webrtc.SurfaceViewRenderer;
45import org.webrtc.VideoTrack;
46
47import java.lang.ref.WeakReference;
48import java.util.Arrays;
49import java.util.Collections;
50import java.util.List;
51import java.util.Set;
52
53import eu.siacs.conversations.Config;
54import eu.siacs.conversations.R;
55import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.services.XmppConnectionService;
61import eu.siacs.conversations.ui.widget.DialpadView;
62import eu.siacs.conversations.ui.util.AvatarWorkerTask;
63import eu.siacs.conversations.ui.util.MainThreadExecutor;
64import eu.siacs.conversations.ui.util.Rationals;
65import eu.siacs.conversations.utils.PermissionUtils;
66import eu.siacs.conversations.utils.TimeFrameUtils;
67import eu.siacs.conversations.xml.Namespace;
68import eu.siacs.conversations.xmpp.Jid;
69import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
70import eu.siacs.conversations.xmpp.jingle.ContentAddition;
71import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
72import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
73import eu.siacs.conversations.xmpp.jingle.Media;
74import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
75
76public class RtpSessionActivity extends XmppActivity
77 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
78 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
79
80 public static final String EXTRA_WITH = "with";
81 public static final String EXTRA_SESSION_ID = "session_id";
82 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
83 public static final String EXTRA_LAST_ACTION = "last_action";
84 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
85 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
86 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
87
88 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
89
90 public static final List<RtpEndUserState> END_CARD =
91 Arrays.asList(
92 RtpEndUserState.APPLICATION_ERROR,
93 RtpEndUserState.SECURITY_ERROR,
94 RtpEndUserState.DECLINED_OR_BUSY,
95 RtpEndUserState.CONNECTIVITY_ERROR,
96 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
97 RtpEndUserState.RETRACTED);
98 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
99 Arrays.asList(
100 RtpEndUserState.APPLICATION_ERROR,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.SECURITY_ERROR);
103 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
104 Arrays.asList(
105 RtpEndUserState.CONNECTING,
106 RtpEndUserState.CONNECTED,
107 RtpEndUserState.RECONNECTING,
108 RtpEndUserState.INCOMING_CONTENT_ADD);
109 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
110 Arrays.asList(
111 RtpEndUserState.CONNECTED,
112 RtpEndUserState.RECONNECTING);
113 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
114 Arrays.asList(
115 RtpEndUserState.ACCEPTING_CALL,
116 RtpEndUserState.CONNECTING,
117 RtpEndUserState.RECONNECTING);
118 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
119 private static final int REQUEST_ACCEPT_CALL = 0x1111;
120 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
121 private static final int REQUEST_ADD_CONTENT = 0x1113;
122 private WeakReference<JingleRtpConnection> rtpConnectionReference;
123
124 private ActivityRtpSessionBinding binding;
125 private PowerManager.WakeLock mProximityWakeLock;
126
127 private final Handler mHandler = new Handler();
128 private final Runnable mTickExecutor =
129 new Runnable() {
130 @Override
131 public void run() {
132 updateCallDuration();
133 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
134 }
135 };
136
137 private static Set<Media> actionToMedia(final String action) {
138 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
139 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
140 } else {
141 return ImmutableSet.of(Media.AUDIO);
142 }
143 }
144
145 private static void addSink(
146 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
147 try {
148 videoTrack.addSink(surfaceViewRenderer);
149 } catch (final IllegalStateException e) {
150 Log.e(
151 Config.LOGTAG,
152 "possible race condition on trying to display video track. ignoring",
153 e);
154 }
155 }
156
157 @Override
158 public void onCreate(Bundle savedInstanceState) {
159 super.onCreate(savedInstanceState);
160 getWindow()
161 .addFlags(
162 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
163 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
164 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
165 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
166 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
167 setSupportActionBar(binding.toolbar);
168
169 binding.dialpad.setClickConsumer(tag -> {
170 final JingleRtpConnection connection =
171 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
172 if (connection != null) connection.applyDtmfTone(tag);
173 });
174
175 if (savedInstanceState != null) {
176 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
177 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
178 }
179 }
180
181 @Override
182 public boolean onCreateOptionsMenu(final Menu menu) {
183 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
184 final MenuItem help = menu.findItem(R.id.action_help);
185 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
186 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
187 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
188 help.setVisible(Config.HELP != null && isHelpButtonVisible());
189 gotoChat.setVisible(isSwitchToConversationVisible());
190 switchToVideo.setVisible(isSwitchToVideoVisible());
191 dialpad.setVisible(isAudioOnlyConversation());
192 return super.onCreateOptionsMenu(menu);
193 }
194
195 @Override
196 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
197 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
198 if (xmppConnectionService != null) {
199 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
200 return true;
201 }
202 }
203 }
204 return super.onKeyDown(keyCode, event);
205 }
206
207 private boolean isHelpButtonVisible() {
208 try {
209 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
210 } catch (IllegalStateException e) {
211 final Intent intent = getIntent();
212 final String state =
213 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
214 if (state != null) {
215 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
216 } else {
217 return false;
218 }
219 }
220 }
221
222 private boolean isSwitchToConversationVisible() {
223 final JingleRtpConnection connection =
224 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
225 return connection != null
226 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
227 }
228
229 private boolean isAudioOnlyConversation() {
230 final JingleRtpConnection connection =
231 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
232
233 return connection != null && !connection.getMedia().contains(Media.VIDEO);
234 }
235
236 private boolean isSwitchToVideoVisible() {
237 final JingleRtpConnection connection =
238 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
239 if (connection == null) {
240 return false;
241 }
242 return connection.isSwitchToVideoAvailable();
243 }
244
245 private void switchToConversation() {
246 final Contact contact = getWith();
247 final Conversation conversation =
248 xmppConnectionService.findOrCreateConversation(
249 contact.getAccount(), contact.getJid(), false, true);
250 switchToConversation(conversation);
251 }
252
253 private void toggleDialpadVisibility() {
254 if (binding.dialpad.getVisibility() == View.VISIBLE) {
255 binding.dialpad.setVisibility(View.GONE);
256 }
257 else {
258 binding.dialpad.setVisibility(View.VISIBLE);
259 }
260 }
261
262 public boolean onOptionsItemSelected(final MenuItem item) {
263 switch (item.getItemId()) {
264 case R.id.action_help:
265 launchHelpInBrowser();
266 return true;
267 case R.id.action_goto_chat:
268 switchToConversation();
269 return true;
270 case R.id.action_dialpad:
271 toggleDialpadVisibility();
272 return true;
273 case R.id.action_switch_to_video:
274 requestPermissionAndSwitchToVideo();
275 return true;
276 }
277 return super.onOptionsItemSelected(item);
278 }
279
280 private void launchHelpInBrowser() {
281 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
282 try {
283 startActivity(intent);
284 } catch (final ActivityNotFoundException e) {
285 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
286 .show();
287 }
288 }
289
290 private void endCall(View view) {
291 endCall();
292 }
293
294 private void endCall() {
295 if (this.rtpConnectionReference == null) {
296 retractSessionProposal();
297 finish();
298 } else {
299 requireRtpConnection().endCall();
300 }
301 }
302
303 private void retractSessionProposal() {
304 final Intent intent = getIntent();
305 final String action = intent.getAction();
306 final Account account = extractAccount(intent);
307 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
308 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
309 if (!Intent.ACTION_VIEW.equals(action)
310 || state == null
311 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
312 resetIntent(
313 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
314 }
315 xmppConnectionService
316 .getJingleConnectionManager()
317 .retractSessionProposal(account, with.asBareJid());
318 }
319
320 private void rejectCall(View view) {
321 requireRtpConnection().rejectCall();
322 finish();
323 }
324
325 private void acceptCall(View view) {
326 requestPermissionsAndAcceptCall();
327 }
328
329 private void acceptContentAdd() {
330 try {
331 requireRtpConnection()
332 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
333 } catch (final IllegalStateException e) {
334 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
335 }
336 }
337
338 private void requestPermissionAndSwitchToVideo() {
339 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
340 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
341 switchToVideo();
342 }
343 }
344
345 private void switchToVideo() {
346 try {
347 requireRtpConnection().addMedia(Media.VIDEO);
348 } catch (final IllegalStateException e) {
349 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
350 }
351 }
352
353 private void acceptContentAdd(final ContentAddition contentAddition) {
354 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
355 Log.d(Config.LOGTAG,"ignore press on content-accept button");
356 return;
357 }
358 requestPermissionAndAcceptContentAdd(contentAddition);
359 }
360
361 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
362 final List<String> permissions = permissions(contentAddition.media());
363 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
364 requireRtpConnection().acceptContentAdd(contentAddition.summary);
365 }
366 }
367
368 private void rejectContentAdd(final View view) {
369 requireRtpConnection().rejectContentAdd();
370 }
371
372 private void requestPermissionsAndAcceptCall() {
373 final List<String> permissions = permissions(getMedia());
374 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
375 putScreenInCallMode();
376 checkRecorderAndAcceptCall();
377 }
378 }
379
380 private List<String> permissions(final Set<Media> media) {
381 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
382 if (media.contains(Media.VIDEO)) {
383 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
384 } else {
385 permissions.add(Manifest.permission.RECORD_AUDIO);
386 }
387 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
388 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
389 }
390 return permissions.build();
391 }
392
393 private void checkRecorderAndAcceptCall() {
394 checkMicrophoneAvailabilityAsync();
395 try {
396 requireRtpConnection().acceptCall();
397 } catch (final IllegalStateException e) {
398 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
399 }
400 }
401
402 private void checkMicrophoneAvailabilityAsync() {
403 new Thread(new MicrophoneAvailabilityCheck(this)).start();
404 }
405
406 private static class MicrophoneAvailabilityCheck implements Runnable {
407
408 private final WeakReference<Activity> activityReference;
409
410 private MicrophoneAvailabilityCheck(final Activity activity) {
411 this.activityReference = new WeakReference<>(activity);
412 }
413
414 @Override
415 public void run() {
416 final long start = SystemClock.elapsedRealtime();
417 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
418 final long stop = SystemClock.elapsedRealtime();
419 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
420 if (isMicrophoneAvailable) {
421 return;
422 }
423 final Activity activity = activityReference.get();
424 if (activity == null) {
425 return;
426 }
427 activity.runOnUiThread(
428 () ->
429 Toast.makeText(
430 activity,
431 R.string.microphone_unavailable,
432 Toast.LENGTH_LONG)
433 .show());
434 }
435 }
436
437 private void putScreenInCallMode() {
438 putScreenInCallMode(requireRtpConnection().getMedia());
439 }
440
441 private void putScreenInCallMode(final Set<Media> media) {
442 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
443 if (!media.contains(Media.VIDEO)) {
444 final JingleRtpConnection rtpConnection =
445 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
446 final AppRTCAudioManager audioManager =
447 rtpConnection == null ? null : rtpConnection.getAudioManager();
448 if (audioManager == null
449 || audioManager.getSelectedAudioDevice()
450 == AppRTCAudioManager.AudioDevice.EARPIECE) {
451 acquireProximityWakeLock();
452 }
453 }
454 }
455
456 @SuppressLint("WakelockTimeout")
457 private void acquireProximityWakeLock() {
458 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
459 if (powerManager == null) {
460 Log.e(Config.LOGTAG, "power manager not available");
461 return;
462 }
463 if (isFinishing()) {
464 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
465 return;
466 }
467 if (this.mProximityWakeLock == null) {
468 this.mProximityWakeLock =
469 powerManager.newWakeLock(
470 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
471 }
472 if (!this.mProximityWakeLock.isHeld()) {
473 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
474 this.mProximityWakeLock.acquire();
475 }
476 }
477
478 private void releaseProximityWakeLock() {
479 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
480 Log.d(Config.LOGTAG, "releasing proximity wake lock");
481 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
482 this.mProximityWakeLock = null;
483 }
484 }
485
486 private void putProximityWakeLockInProperState(
487 final AppRTCAudioManager.AudioDevice audioDevice) {
488 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
489 acquireProximityWakeLock();
490 } else {
491 releaseProximityWakeLock();
492 }
493 }
494
495 @Override
496 protected void refreshUiReal() {}
497
498 @Override
499 public void onNewIntent(final Intent intent) {
500 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
501 super.onNewIntent(intent);
502 setIntent(intent);
503 if (xmppConnectionService == null) {
504 Log.d(
505 Config.LOGTAG,
506 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
507 return;
508 }
509 final Account account = extractAccount(intent);
510 final String action = intent.getAction();
511 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
512 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
513 if (sessionId != null) {
514 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
515 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
516 return;
517 }
518 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
519 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
520 requestPermissionsAndAcceptCall();
521 resetIntent(intent.getExtras());
522 }
523 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
524 proposeJingleRtpSession(account, with, actionToMedia(action));
525 setWith(account.getRoster().getContact(with), null);
526 } else {
527 throw new IllegalStateException("received onNewIntent without sessionId");
528 }
529 }
530
531 @Override
532 void onBackendConnected() {
533 final Intent intent = getIntent();
534 final String action = intent.getAction();
535 final Account account = extractAccount(intent);
536 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
537 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
538 if (sessionId != null) {
539 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
540 return;
541 }
542 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
543 Log.d(Config.LOGTAG, "intent action was accept");
544 requestPermissionsAndAcceptCall();
545 resetIntent(intent.getExtras());
546 }
547 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
548 proposeJingleRtpSession(account, with, actionToMedia(action));
549 setWith(account.getRoster().getContact(with), null);
550 } else if (Intent.ACTION_VIEW.equals(action)) {
551 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
552 final RtpEndUserState state =
553 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
554 if (state != null) {
555 Log.d(Config.LOGTAG, "restored last state from intent extra");
556 updateButtonConfiguration(state);
557 updateVerifiedShield(false);
558 updateStateDisplay(state);
559 updateIncomingCallScreen(state);
560 invalidateOptionsMenu();
561 }
562 setWith(account.getRoster().getContact(with), state);
563 if (xmppConnectionService
564 .getJingleConnectionManager()
565 .fireJingleRtpConnectionStateUpdates()) {
566 return;
567 }
568 if (END_CARD.contains(state)
569 || xmppConnectionService
570 .getJingleConnectionManager()
571 .hasMatchingProposal(account, with)) {
572 return;
573 }
574 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
575 finish();
576 }
577 }
578
579 private void setWith(final RtpEndUserState state) {
580 setWith(getWith(), state);
581 }
582
583 private void setWith(final Contact contact, final RtpEndUserState state) {
584 binding.with.setText(contact.getDisplayName());
585 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
586 .contains(state)) {
587 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
588 binding.withJid.setVisibility(View.VISIBLE);
589 } else {
590 binding.withJid.setVisibility(View.GONE);
591 }
592 }
593
594 private void proposeJingleRtpSession(
595 final Account account, final Jid with, final Set<Media> media) {
596 checkMicrophoneAvailabilityAsync();
597 if (with.isBareJid()) {
598 xmppConnectionService
599 .getJingleConnectionManager()
600 .proposeJingleRtpSession(account, with, media);
601 } else {
602 final String sessionId =
603 xmppConnectionService
604 .getJingleConnectionManager()
605 .initializeRtpSession(account, with, media);
606 initializeActivityWithRunningRtpSession(account, with, sessionId);
607 resetIntent(account, with, sessionId);
608 }
609 putScreenInCallMode(media);
610 }
611
612 @Override
613 public void onRequestPermissionsResult(
614 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
615 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
616 final PermissionUtils.PermissionResult permissionResult =
617 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
618 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
619 if (requestCode == REQUEST_ACCEPT_CALL) {
620 checkRecorderAndAcceptCall();
621 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
622 acceptContentAdd();
623 } else if (requestCode == REQUEST_ADD_CONTENT) {
624 switchToVideo();
625 }
626 } else {
627 @StringRes int res;
628 final String firstDenied =
629 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
630 if (firstDenied == null) {
631 return;
632 }
633 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
634 res = R.string.no_microphone_permission;
635 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
636 res = R.string.no_camera_permission;
637 } else {
638 throw new IllegalStateException("Invalid permission result request");
639 }
640 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
641 .show();
642 }
643 }
644
645 @Override
646 public void onStart() {
647 super.onStart();
648 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
649 this.binding.remoteVideo.setOnAspectRatioChanged(this);
650 }
651
652 @Override
653 public void onStop() {
654 mHandler.removeCallbacks(mTickExecutor);
655 binding.remoteVideo.release();
656 binding.remoteVideo.setOnAspectRatioChanged(null);
657 binding.localVideo.release();
658 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
659 final JingleRtpConnection jingleRtpConnection =
660 weakReference == null ? null : weakReference.get();
661 if (jingleRtpConnection != null) {
662 releaseVideoTracks(jingleRtpConnection);
663 }
664 releaseProximityWakeLock();
665 super.onStop();
666 }
667
668 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
669 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
670 if (remoteVideo.isPresent()) {
671 remoteVideo.get().removeSink(binding.remoteVideo);
672 }
673 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
674 if (localVideo.isPresent()) {
675 localVideo.get().removeSink(binding.localVideo);
676 }
677 }
678
679 @Override
680 public void onBackPressed() {
681 if (isConnected()) {
682 if (switchToPictureInPicture()) {
683 return;
684 }
685 } else {
686 endCall();
687 }
688 super.onBackPressed();
689 }
690
691 @Override
692 public void onUserLeaveHint() {
693 super.onUserLeaveHint();
694 if (switchToPictureInPicture()) {
695 return;
696 }
697 // TODO apparently this method is not getting called on Android 10 when using the task
698 // switcher
699 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
700 retractSessionProposal();
701 }
702 }
703
704 private boolean isConnected() {
705 final JingleRtpConnection connection =
706 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
707 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
708 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
709 }
710
711 private boolean switchToPictureInPicture() {
712 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
713 if (shouldBePictureInPicture()) {
714 startPictureInPicture();
715 return true;
716 }
717 }
718 return false;
719 }
720
721 @RequiresApi(api = Build.VERSION_CODES.O)
722 private void startPictureInPicture() {
723 try {
724 final Rational rational = this.binding.remoteVideo.getAspectRatio();
725 final Rational clippedRational = Rationals.clip(rational);
726 Log.d(
727 Config.LOGTAG,
728 "suggested rational " + rational + ". clipped to " + clippedRational);
729 enterPictureInPictureMode(
730 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
731 } catch (final IllegalStateException e) {
732 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
733 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
734 }
735 }
736
737 @Override
738 public void onAspectRatioChanged(final Rational rational) {
739 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
740 final Rational clippedRational = Rationals.clip(rational);
741 Log.d(
742 Config.LOGTAG,
743 "suggested rational after aspect ratio change "
744 + rational
745 + ". clipped to "
746 + clippedRational);
747 setPictureInPictureParams(
748 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
749 }
750 }
751
752 private boolean deviceSupportsPictureInPicture() {
753 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
754 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
755 } else {
756 return false;
757 }
758 }
759
760 private boolean shouldBePictureInPicture() {
761 try {
762 final JingleRtpConnection rtpConnection = requireRtpConnection();
763 return rtpConnection.getMedia().contains(Media.VIDEO)
764 && Arrays.asList(
765 RtpEndUserState.ACCEPTING_CALL,
766 RtpEndUserState.CONNECTING,
767 RtpEndUserState.CONNECTED)
768 .contains(rtpConnection.getEndUserState());
769 } catch (final IllegalStateException e) {
770 return false;
771 }
772 }
773
774 private boolean initializeActivityWithRunningRtpSession(
775 final Account account, Jid with, String sessionId) {
776 final WeakReference<JingleRtpConnection> reference =
777 xmppConnectionService
778 .getJingleConnectionManager()
779 .findJingleRtpConnection(account, with, sessionId);
780 if (reference == null || reference.get() == null) {
781 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
782 xmppConnectionService
783 .getJingleConnectionManager()
784 .getTerminalSessionState(with, sessionId);
785 if (terminatedRtpSession == null) {
786 throw new IllegalStateException(
787 "failed to initialize activity with running rtp session. session not found");
788 }
789 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
790 return true;
791 }
792 this.rtpConnectionReference = reference;
793 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
794 final boolean verified = requireRtpConnection().isVerified();
795 if (currentState == RtpEndUserState.ENDED) {
796 finish();
797 return true;
798 }
799 final Set<Media> media = getMedia();
800 final ContentAddition contentAddition = getPendingContentAddition();
801 if (currentState == RtpEndUserState.INCOMING_CALL) {
802 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
803 }
804 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
805 requireRtpConnection().getState())) {
806 putScreenInCallMode();
807 }
808 setWith(currentState);
809 updateVideoViews(currentState);
810 updateStateDisplay(currentState, media, contentAddition);
811 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
812 updateButtonConfiguration(currentState, media, contentAddition);
813 updateIncomingCallScreen(currentState);
814 invalidateOptionsMenu();
815 return false;
816 }
817
818 private void initializeWithTerminatedSessionState(
819 final Account account,
820 final Jid with,
821 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
822 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
823 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
824 finish();
825 return;
826 }
827 final RtpEndUserState state = terminatedRtpSession.state;
828 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
829 updateButtonConfiguration(state);
830 updateStateDisplay(state);
831 updateIncomingCallScreen(state);
832 updateCallDuration();
833 updateVerifiedShield(false);
834 invalidateOptionsMenu();
835 setWith(account.getRoster().getContact(with), state);
836 }
837
838 private void reInitializeActivityWithRunningRtpSession(
839 final Account account, Jid with, String sessionId) {
840 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
841 resetIntent(account, with, sessionId);
842 }
843
844 private void resetIntent(final Account account, final Jid with, final String sessionId) {
845 final Intent intent = new Intent(Intent.ACTION_VIEW);
846 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
847 intent.putExtra(EXTRA_WITH, with.toEscapedString());
848 intent.putExtra(EXTRA_SESSION_ID, sessionId);
849 setIntent(intent);
850 }
851
852 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
853 surfaceViewRenderer.setVisibility(View.VISIBLE);
854 try {
855 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
856 } catch (final IllegalStateException e) {
857 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
858 }
859 surfaceViewRenderer.setEnableHardwareScaler(true);
860 }
861
862 private void updateStateDisplay(final RtpEndUserState state) {
863 updateStateDisplay(state, Collections.emptySet(), null);
864 }
865
866 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
867 switch (state) {
868 case INCOMING_CALL:
869 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
870 if (media.contains(Media.VIDEO)) {
871 setTitle(R.string.rtp_state_incoming_video_call);
872 } else {
873 setTitle(R.string.rtp_state_incoming_call);
874 }
875 break;
876 case INCOMING_CONTENT_ADD:
877 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
878 setTitle(R.string.rtp_state_content_add_video);
879 } else {
880 setTitle(R.string.rtp_state_content_add);
881 }
882 break;
883 case CONNECTING:
884 setTitle(R.string.rtp_state_connecting);
885 break;
886 case CONNECTED:
887 setTitle(R.string.rtp_state_connected);
888 break;
889 case RECONNECTING:
890 setTitle(R.string.rtp_state_reconnecting);
891 break;
892 case ACCEPTING_CALL:
893 setTitle(R.string.rtp_state_accepting_call);
894 break;
895 case ENDING_CALL:
896 setTitle(R.string.rtp_state_ending_call);
897 break;
898 case FINDING_DEVICE:
899 setTitle(R.string.rtp_state_finding_device);
900 break;
901 case RINGING:
902 setTitle(R.string.rtp_state_ringing);
903 break;
904 case DECLINED_OR_BUSY:
905 setTitle(R.string.rtp_state_declined_or_busy);
906 break;
907 case CONNECTIVITY_ERROR:
908 setTitle(R.string.rtp_state_connectivity_error);
909 break;
910 case CONNECTIVITY_LOST_ERROR:
911 setTitle(R.string.rtp_state_connectivity_lost_error);
912 break;
913 case RETRACTED:
914 setTitle(R.string.rtp_state_retracted);
915 break;
916 case APPLICATION_ERROR:
917 setTitle(R.string.rtp_state_application_failure);
918 break;
919 case SECURITY_ERROR:
920 setTitle(R.string.rtp_state_security_error);
921 break;
922 case ENDED:
923 throw new IllegalStateException(
924 "Activity should have called finishAndReleaseWakeLock();");
925 default:
926 throw new IllegalStateException(
927 String.format("State %s has not been handled in UI", state));
928 }
929 }
930
931 private void updateVerifiedShield(final boolean verified) {
932 if (isPictureInPicture()) {
933 this.binding.verified.setVisibility(View.GONE);
934 return;
935 }
936 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
937 }
938
939 private void updateIncomingCallScreen(final RtpEndUserState state) {
940 updateIncomingCallScreen(state, null);
941 }
942
943 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
944 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
945 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
946 if (show) {
947 binding.contactPhoto.setVisibility(View.VISIBLE);
948 if (contact == null) {
949 AvatarWorkerTask.loadAvatar(
950 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
951 } else {
952 AvatarWorkerTask.loadAvatar(
953 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
954 }
955 } else {
956 binding.contactPhoto.setVisibility(View.GONE);
957 }
958 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
959 binding.usingAccount.setVisibility(View.VISIBLE);
960 binding.usingAccount.setText(
961 getString(
962 R.string.using_account,
963 account.getJid().asBareJid().toEscapedString()));
964 } else {
965 binding.usingAccount.setVisibility(View.GONE);
966 binding.contactPhoto.setVisibility(View.GONE);
967 }
968 }
969
970 private Set<Media> getMedia() {
971 return requireRtpConnection().getMedia();
972 }
973
974 public ContentAddition getPendingContentAddition() {
975 return requireRtpConnection().getPendingContentAddition();
976 }
977
978 private void updateButtonConfiguration(final RtpEndUserState state) {
979 updateButtonConfiguration(state, Collections.emptySet(), null);
980 }
981
982 @SuppressLint("RestrictedApi")
983 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
984 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
985 this.binding.rejectCall.setVisibility(View.INVISIBLE);
986 this.binding.endCall.setVisibility(View.INVISIBLE);
987 this.binding.acceptCall.setVisibility(View.INVISIBLE);
988 } else if (state == RtpEndUserState.INCOMING_CALL) {
989 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
990 this.binding.rejectCall.setOnClickListener(this::rejectCall);
991 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
992 this.binding.rejectCall.setVisibility(View.VISIBLE);
993 this.binding.endCall.setVisibility(View.INVISIBLE);
994 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
995 this.binding.acceptCall.setOnClickListener(this::acceptCall);
996 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
997 this.binding.acceptCall.setVisibility(View.VISIBLE);
998 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
999 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
1000 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
1001 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1002 this.binding.rejectCall.setVisibility(View.VISIBLE);
1003 this.binding.endCall.setVisibility(View.INVISIBLE);
1004 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1005 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1006 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
1007 this.binding.acceptCall.setVisibility(View.VISIBLE);
1008 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
1009 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1010 this.binding.rejectCall.setOnClickListener(this::exit);
1011 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1012 this.binding.rejectCall.setVisibility(View.VISIBLE);
1013 this.binding.endCall.setVisibility(View.INVISIBLE);
1014 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1015 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1016 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1017 this.binding.acceptCall.setVisibility(View.VISIBLE);
1018 } else if (asList(
1019 RtpEndUserState.CONNECTIVITY_ERROR,
1020 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1021 RtpEndUserState.APPLICATION_ERROR,
1022 RtpEndUserState.RETRACTED,
1023 RtpEndUserState.SECURITY_ERROR)
1024 .contains(state)) {
1025 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1026 this.binding.rejectCall.setOnClickListener(this::exit);
1027 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1028 this.binding.rejectCall.setVisibility(View.VISIBLE);
1029 this.binding.endCall.setVisibility(View.INVISIBLE);
1030 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1031 this.binding.acceptCall.setOnClickListener(this::retry);
1032 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1033 this.binding.acceptCall.setVisibility(View.VISIBLE);
1034 } else {
1035 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1036 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1037 this.binding.endCall.setOnClickListener(this::endCall);
1038 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1039 this.binding.endCall.setVisibility(View.VISIBLE);
1040 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1041 }
1042 updateInCallButtonConfiguration(state, media);
1043 }
1044
1045 private boolean isPictureInPicture() {
1046 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1047 return isInPictureInPictureMode();
1048 } else {
1049 return false;
1050 }
1051 }
1052
1053 private void updateInCallButtonConfiguration() {
1054 updateInCallButtonConfiguration(
1055 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1056 }
1057
1058 @SuppressLint("RestrictedApi")
1059 private void updateInCallButtonConfiguration(
1060 final RtpEndUserState state, final Set<Media> media) {
1061 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1062 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1063 if (media.contains(Media.VIDEO)) {
1064 final JingleRtpConnection rtpConnection = requireRtpConnection();
1065 updateInCallButtonConfigurationVideo(
1066 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1067 } else {
1068 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1069 updateInCallButtonConfigurationSpeaker(
1070 audioManager.getSelectedAudioDevice(),
1071 audioManager.getAudioDevices().size());
1072 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1073 }
1074 if (media.contains(Media.AUDIO)) {
1075 updateInCallButtonConfigurationMicrophone(
1076 requireRtpConnection().isMicrophoneEnabled());
1077 } else {
1078 this.binding.inCallActionLeft.setVisibility(View.GONE);
1079 }
1080 } else {
1081 this.binding.inCallActionLeft.setVisibility(View.GONE);
1082 this.binding.inCallActionRight.setVisibility(View.GONE);
1083 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1084 }
1085 }
1086
1087 @SuppressLint("RestrictedApi")
1088 private void updateInCallButtonConfigurationSpeaker(
1089 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1090 switch (selectedAudioDevice) {
1091 case EARPIECE:
1092 this.binding.inCallActionRight.setImageResource(
1093 R.drawable.ic_volume_off_black_24dp);
1094 if (numberOfChoices >= 2) {
1095 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1096 } else {
1097 this.binding.inCallActionRight.setOnClickListener(null);
1098 this.binding.inCallActionRight.setClickable(false);
1099 }
1100 break;
1101 case WIRED_HEADSET:
1102 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1103 this.binding.inCallActionRight.setOnClickListener(null);
1104 this.binding.inCallActionRight.setClickable(false);
1105 break;
1106 case SPEAKER_PHONE:
1107 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1108 if (numberOfChoices >= 2) {
1109 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1110 } else {
1111 this.binding.inCallActionRight.setOnClickListener(null);
1112 this.binding.inCallActionRight.setClickable(false);
1113 }
1114 break;
1115 case BLUETOOTH:
1116 this.binding.inCallActionRight.setImageResource(
1117 R.drawable.ic_bluetooth_audio_black_24dp);
1118 this.binding.inCallActionRight.setOnClickListener(null);
1119 this.binding.inCallActionRight.setClickable(false);
1120 break;
1121 }
1122 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1123 }
1124
1125 @SuppressLint("RestrictedApi")
1126 private void updateInCallButtonConfigurationVideo(
1127 final boolean videoEnabled, final boolean isCameraSwitchable) {
1128 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1129 if (isCameraSwitchable) {
1130 this.binding.inCallActionFarRight.setImageResource(
1131 R.drawable.ic_flip_camera_android_black_24dp);
1132 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1133 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1134 } else {
1135 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1136 }
1137 if (videoEnabled) {
1138 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1139 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1140 } else {
1141 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1142 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1143 }
1144 }
1145
1146 private void switchCamera(final View view) {
1147 Futures.addCallback(
1148 requireRtpConnection().switchCamera(),
1149 new FutureCallback<Boolean>() {
1150 @Override
1151 public void onSuccess(@Nullable Boolean isFrontCamera) {
1152 binding.localVideo.setMirror(isFrontCamera);
1153 }
1154
1155 @Override
1156 public void onFailure(@NonNull final Throwable throwable) {
1157 Log.d(
1158 Config.LOGTAG,
1159 "could not switch camera",
1160 Throwables.getRootCause(throwable));
1161 Toast.makeText(
1162 RtpSessionActivity.this,
1163 R.string.could_not_switch_camera,
1164 Toast.LENGTH_LONG)
1165 .show();
1166 }
1167 },
1168 MainThreadExecutor.getInstance());
1169 }
1170
1171 private void enableVideo(View view) {
1172 try {
1173 requireRtpConnection().setVideoEnabled(true);
1174 } catch (final IllegalStateException e) {
1175 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1176 return;
1177 }
1178 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1179 }
1180
1181 private void disableVideo(View view) {
1182 final JingleRtpConnection rtpConnection = requireRtpConnection();
1183 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1184 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1185 rtpConnection.retractContentAdd();
1186 return;
1187 }
1188 requireRtpConnection().setVideoEnabled(false);
1189 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1190 }
1191
1192 @SuppressLint("RestrictedApi")
1193 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1194 if (microphoneEnabled) {
1195 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1196 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1197 } else {
1198 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1199 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1200 }
1201 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1202 }
1203
1204 private void updateCallDuration() {
1205 final JingleRtpConnection connection =
1206 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1207 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1208 this.binding.duration.setVisibility(View.GONE);
1209 return;
1210 }
1211 if (connection.zeroDuration()) {
1212 this.binding.duration.setVisibility(View.GONE);
1213 } else {
1214 this.binding.duration.setText(
1215 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1216 this.binding.duration.setVisibility(View.VISIBLE);
1217 }
1218 }
1219
1220 private void updateVideoViews(final RtpEndUserState state) {
1221 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1222 binding.localVideo.setVisibility(View.GONE);
1223 binding.localVideo.release();
1224 binding.remoteVideoWrapper.setVisibility(View.GONE);
1225 binding.remoteVideo.release();
1226 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1227 if (isPictureInPicture()) {
1228 binding.appBarLayout.setVisibility(View.GONE);
1229 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1230 if (Arrays.asList(
1231 RtpEndUserState.APPLICATION_ERROR,
1232 RtpEndUserState.CONNECTIVITY_ERROR,
1233 RtpEndUserState.SECURITY_ERROR)
1234 .contains(state)) {
1235 binding.pipWarning.setVisibility(View.VISIBLE);
1236 binding.pipWaiting.setVisibility(View.GONE);
1237 } else {
1238 binding.pipWarning.setVisibility(View.GONE);
1239 binding.pipWaiting.setVisibility(View.GONE);
1240 }
1241 } else {
1242 binding.appBarLayout.setVisibility(View.VISIBLE);
1243 binding.pipPlaceholder.setVisibility(View.GONE);
1244 }
1245 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1246 return;
1247 }
1248 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1249 binding.localVideo.setVisibility(View.GONE);
1250 binding.remoteVideoWrapper.setVisibility(View.GONE);
1251 binding.appBarLayout.setVisibility(View.GONE);
1252 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1253 binding.pipWarning.setVisibility(View.GONE);
1254 binding.pipWaiting.setVisibility(View.VISIBLE);
1255 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1256 return;
1257 }
1258 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1259 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1260 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1261 // paint local view over remote view
1262 binding.localVideo.setZOrderMediaOverlay(true);
1263 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1264 addSink(localVideoTrack.get(), binding.localVideo);
1265 } else {
1266 binding.localVideo.setVisibility(View.GONE);
1267 }
1268 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1269 if (remoteVideoTrack.isPresent()) {
1270 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1271 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1272 binding.remoteVideo.setScalingType(
1273 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1274 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1275 if (state == RtpEndUserState.CONNECTED) {
1276 binding.appBarLayout.setVisibility(View.GONE);
1277 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1278 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1279 } else {
1280 binding.appBarLayout.setVisibility(View.VISIBLE);
1281 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1282 binding.remoteVideoWrapper.setVisibility(View.GONE);
1283 }
1284 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1285 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1286 } else {
1287 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1288 }
1289 } else {
1290 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1291 binding.remoteVideoWrapper.setVisibility(View.GONE);
1292 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1293 }
1294 }
1295
1296 private Optional<VideoTrack> getLocalVideoTrack() {
1297 final JingleRtpConnection connection =
1298 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1299 if (connection == null) {
1300 return Optional.absent();
1301 }
1302 return connection.getLocalVideoTrack();
1303 }
1304
1305 private Optional<VideoTrack> getRemoteVideoTrack() {
1306 final JingleRtpConnection connection =
1307 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1308 if (connection == null) {
1309 return Optional.absent();
1310 }
1311 return connection.getRemoteVideoTrack();
1312 }
1313
1314 private void disableMicrophone(View view) {
1315 final JingleRtpConnection rtpConnection = requireRtpConnection();
1316 if (rtpConnection.setMicrophoneEnabled(false)) {
1317 updateInCallButtonConfiguration();
1318 }
1319 }
1320
1321 private void enableMicrophone(View view) {
1322 final JingleRtpConnection rtpConnection = requireRtpConnection();
1323 if (rtpConnection.setMicrophoneEnabled(true)) {
1324 updateInCallButtonConfiguration();
1325 }
1326 }
1327
1328 private void switchToEarpiece(View view) {
1329 requireRtpConnection()
1330 .getAudioManager()
1331 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1332 acquireProximityWakeLock();
1333 }
1334
1335 private void switchToSpeaker(View view) {
1336 requireRtpConnection()
1337 .getAudioManager()
1338 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1339 releaseProximityWakeLock();
1340 }
1341
1342 private void retry(View view) {
1343 final Intent intent = getIntent();
1344 final Account account = extractAccount(intent);
1345 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1346 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1347 final String action = intent.getAction();
1348 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1349 this.rtpConnectionReference = null;
1350 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1351 proposeJingleRtpSession(account, with, media);
1352 }
1353
1354 private void exit(final View view) {
1355 finish();
1356 }
1357
1358 private void recordVoiceMail(final View view) {
1359 final Intent intent = getIntent();
1360 final Account account = extractAccount(intent);
1361 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1362 final Conversation conversation =
1363 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1364 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1365 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1366 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1367 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1368 launchIntent.putExtra(
1369 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1370 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1371 startActivity(launchIntent);
1372 finish();
1373 }
1374
1375 private Contact getWith() {
1376 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1377 final Account account = id.account;
1378 return account.getRoster().getContact(id.with);
1379 }
1380
1381 private JingleRtpConnection requireRtpConnection() {
1382 final JingleRtpConnection connection =
1383 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1384 if (connection == null) {
1385 throw new IllegalStateException("No RTP connection found");
1386 }
1387 return connection;
1388 }
1389
1390 @Override
1391 public void onJingleRtpConnectionUpdate(
1392 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1393 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1394 if (END_CARD.contains(state)) {
1395 Log.d(Config.LOGTAG, "end card reached");
1396 releaseProximityWakeLock();
1397 runOnUiThread(
1398 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1399 }
1400 if (with.isBareJid()) {
1401 updateRtpSessionProposalState(account, with, state);
1402 return;
1403 }
1404 if (emptyReference(this.rtpConnectionReference)) {
1405 if (END_CARD.contains(state)) {
1406 Log.d(Config.LOGTAG, "not reinitializing session");
1407 return;
1408 }
1409 // this happens when going from proposed session to actual session
1410 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1411 return;
1412 }
1413 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1414 final boolean verified = requireRtpConnection().isVerified();
1415 final Set<Media> media = getMedia();
1416 final ContentAddition contentAddition = getPendingContentAddition();
1417 final Contact contact = getWith();
1418 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1419 if (state == RtpEndUserState.ENDED) {
1420 finish();
1421 return;
1422 }
1423 runOnUiThread(
1424 () -> {
1425 updateStateDisplay(state, media, contentAddition);
1426 updateVerifiedShield(
1427 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1428 updateButtonConfiguration(state, media, contentAddition);
1429 updateVideoViews(state);
1430 updateIncomingCallScreen(state, contact);
1431 invalidateOptionsMenu();
1432 });
1433 if (END_CARD.contains(state)) {
1434 final JingleRtpConnection rtpConnection = requireRtpConnection();
1435 resetIntent(account, with, state, rtpConnection.getMedia());
1436 releaseVideoTracks(rtpConnection);
1437 this.rtpConnectionReference = null;
1438 }
1439 } else {
1440 Log.d(Config.LOGTAG, "received update for other rtp session");
1441 }
1442 }
1443
1444 @Override
1445 public void onAudioDeviceChanged(
1446 AppRTCAudioManager.AudioDevice selectedAudioDevice,
1447 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1448 Log.d(
1449 Config.LOGTAG,
1450 "onAudioDeviceChanged in activity: selected:"
1451 + selectedAudioDevice
1452 + ", available:"
1453 + availableAudioDevices);
1454 try {
1455 if (getMedia().contains(Media.VIDEO)) {
1456 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1457 return;
1458 }
1459 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1460 if (endUserState == RtpEndUserState.CONNECTED) {
1461 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1462 updateInCallButtonConfigurationSpeaker(
1463 audioManager.getSelectedAudioDevice(),
1464 audioManager.getAudioDevices().size());
1465 } else if (END_CARD.contains(endUserState)) {
1466 Log.d(
1467 Config.LOGTAG,
1468 "onAudioDeviceChanged() nothing to do because end card has been reached");
1469 } else {
1470 putProximityWakeLockInProperState(selectedAudioDevice);
1471 }
1472 } catch (IllegalStateException e) {
1473 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1474 }
1475 }
1476
1477 @Override
1478 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1479 super.onSaveInstanceState(outState);
1480 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1481 }
1482
1483 private void updateRtpSessionProposalState(
1484 final Account account, final Jid with, final RtpEndUserState state) {
1485 final Intent currentIntent = getIntent();
1486 final String withExtra =
1487 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1488 if (withExtra == null) {
1489 return;
1490 }
1491 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1492 runOnUiThread(
1493 () -> {
1494 updateVerifiedShield(false);
1495 updateStateDisplay(state);
1496 updateButtonConfiguration(state);
1497 updateIncomingCallScreen(state);
1498 invalidateOptionsMenu();
1499 });
1500 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1501 }
1502 }
1503
1504 private void resetIntent(final Bundle extras) {
1505 final Intent intent = new Intent(Intent.ACTION_VIEW);
1506 intent.putExtras(extras);
1507 setIntent(intent);
1508 }
1509
1510 private void resetIntent(
1511 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1512 final Intent intent = new Intent(Intent.ACTION_VIEW);
1513 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1514 if (account.getRoster()
1515 .getContact(with)
1516 .getPresences()
1517 .anySupport(Namespace.JINGLE_MESSAGE)) {
1518 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1519 } else {
1520 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1521 }
1522 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1523 intent.putExtra(
1524 EXTRA_LAST_ACTION,
1525 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1526 setIntent(intent);
1527 }
1528
1529 private static boolean emptyReference(final WeakReference<?> weakReference) {
1530 return weakReference == null || weakReference.get() == null;
1531 }
1532}