1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.os.Build;
16import android.os.Bundle;
17import android.os.Handler;
18import android.os.PowerManager;
19import android.os.SystemClock;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import org.jetbrains.annotations.NotNull;
44import org.webrtc.RendererCommon;
45import org.webrtc.SurfaceViewRenderer;
46import org.webrtc.VideoTrack;
47
48import java.lang.ref.WeakReference;
49import java.util.Arrays;
50import java.util.Collections;
51import java.util.List;
52import java.util.Set;
53
54import eu.siacs.conversations.Config;
55import eu.siacs.conversations.R;
56import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
57import eu.siacs.conversations.entities.Account;
58import eu.siacs.conversations.entities.Contact;
59import eu.siacs.conversations.entities.Conversation;
60import eu.siacs.conversations.services.AppRTCAudioManager;
61import eu.siacs.conversations.services.XmppConnectionService;
62import eu.siacs.conversations.ui.widget.DialpadView;
63import eu.siacs.conversations.ui.util.AvatarWorkerTask;
64import eu.siacs.conversations.ui.util.MainThreadExecutor;
65import eu.siacs.conversations.ui.util.Rationals;
66import eu.siacs.conversations.utils.PermissionUtils;
67import eu.siacs.conversations.utils.TimeFrameUtils;
68import eu.siacs.conversations.xml.Namespace;
69import eu.siacs.conversations.xmpp.Jid;
70import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
71import eu.siacs.conversations.xmpp.jingle.ContentAddition;
72import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
73import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
74import eu.siacs.conversations.xmpp.jingle.Media;
75import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
76
77public class RtpSessionActivity extends XmppActivity
78 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
79 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
80
81 public static final String EXTRA_WITH = "with";
82 public static final String EXTRA_SESSION_ID = "session_id";
83 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
84 public static final String EXTRA_LAST_ACTION = "last_action";
85 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
86 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
87 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
88
89 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
90
91 public static final List<RtpEndUserState> END_CARD =
92 Arrays.asList(
93 RtpEndUserState.APPLICATION_ERROR,
94 RtpEndUserState.SECURITY_ERROR,
95 RtpEndUserState.DECLINED_OR_BUSY,
96 RtpEndUserState.CONNECTIVITY_ERROR,
97 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
98 RtpEndUserState.RETRACTED);
99 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
100 Arrays.asList(
101 RtpEndUserState.APPLICATION_ERROR,
102 RtpEndUserState.CONNECTIVITY_ERROR,
103 RtpEndUserState.SECURITY_ERROR);
104 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
105 Arrays.asList(
106 RtpEndUserState.CONNECTING,
107 RtpEndUserState.CONNECTED,
108 RtpEndUserState.RECONNECTING,
109 RtpEndUserState.INCOMING_CONTENT_ADD);
110 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
111 Arrays.asList(
112 RtpEndUserState.CONNECTED,
113 RtpEndUserState.RECONNECTING);
114 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
115 Arrays.asList(
116 RtpEndUserState.ACCEPTING_CALL,
117 RtpEndUserState.CONNECTING,
118 RtpEndUserState.RECONNECTING);
119 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
120 private static final int REQUEST_ACCEPT_CALL = 0x1111;
121 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
122 private static final int REQUEST_ADD_CONTENT = 0x1113;
123 private WeakReference<JingleRtpConnection> rtpConnectionReference;
124
125 private ActivityRtpSessionBinding binding;
126 private PowerManager.WakeLock mProximityWakeLock;
127
128 private final Handler mHandler = new Handler();
129 private final Runnable mTickExecutor =
130 new Runnable() {
131 @Override
132 public void run() {
133 updateCallDuration();
134 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
135 }
136 };
137
138 private static Set<Media> actionToMedia(final String action) {
139 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
140 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
141 } else {
142 return ImmutableSet.of(Media.AUDIO);
143 }
144 }
145
146 private static void addSink(
147 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
148 try {
149 videoTrack.addSink(surfaceViewRenderer);
150 } catch (final IllegalStateException e) {
151 Log.e(
152 Config.LOGTAG,
153 "possible race condition on trying to display video track. ignoring",
154 e);
155 }
156 }
157
158 @Override
159 public void onCreate(Bundle savedInstanceState) {
160 super.onCreate(savedInstanceState);
161 getWindow()
162 .addFlags(
163 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
164 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
165 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
166 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
167 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
168 setSupportActionBar(binding.toolbar);
169
170 binding.dialpad.setClickConsumer(tag -> {
171 final JingleRtpConnection connection =
172 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
173 if (connection != null) connection.applyDtmfTone(tag);
174 });
175
176 if (savedInstanceState != null) {
177 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
178 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
179 }
180 }
181
182 @Override
183 public boolean onCreateOptionsMenu(final Menu menu) {
184 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
185 final MenuItem help = menu.findItem(R.id.action_help);
186 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
187 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
188 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
189 help.setVisible(Config.HELP != null && isHelpButtonVisible());
190 gotoChat.setVisible(isSwitchToConversationVisible());
191 switchToVideo.setVisible(isSwitchToVideoVisible());
192 dialpad.setVisible(isAudioOnlyConversation());
193 return super.onCreateOptionsMenu(menu);
194 }
195
196 @Override
197 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
198 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
199 if (xmppConnectionService != null) {
200 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
201 return true;
202 }
203 }
204 }
205 return super.onKeyDown(keyCode, event);
206 }
207
208 private boolean isHelpButtonVisible() {
209 try {
210 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
211 } catch (IllegalStateException e) {
212 final Intent intent = getIntent();
213 final String state =
214 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
215 if (state != null) {
216 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
217 } else {
218 return false;
219 }
220 }
221 }
222
223 private boolean isSwitchToConversationVisible() {
224 final JingleRtpConnection connection =
225 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
226 return connection != null
227 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
228 }
229
230 private boolean isAudioOnlyConversation() {
231 final JingleRtpConnection connection =
232 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
233
234 return connection != null && !connection.getMedia().contains(Media.VIDEO);
235 }
236
237 private boolean isSwitchToVideoVisible() {
238 final JingleRtpConnection connection =
239 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
240 if (connection == null) {
241 return false;
242 }
243 return connection.isSwitchToVideoAvailable();
244 }
245
246 private void switchToConversation() {
247 final Contact contact = getWith();
248 final Conversation conversation =
249 xmppConnectionService.findOrCreateConversation(
250 contact.getAccount(), contact.getJid(), false, true);
251 switchToConversation(conversation);
252 }
253
254 private void toggleDialpadVisibility() {
255 if (binding.dialpad.getVisibility() == View.VISIBLE) {
256 binding.dialpad.setVisibility(View.GONE);
257 }
258 else {
259 binding.dialpad.setVisibility(View.VISIBLE);
260 }
261 }
262
263 public boolean onOptionsItemSelected(final MenuItem item) {
264 switch (item.getItemId()) {
265 case R.id.action_help:
266 launchHelpInBrowser();
267 return true;
268 case R.id.action_goto_chat:
269 switchToConversation();
270 return true;
271 case R.id.action_dialpad:
272 toggleDialpadVisibility();
273 return true;
274 case R.id.action_switch_to_video:
275 requestPermissionAndSwitchToVideo();
276 return true;
277 }
278 return super.onOptionsItemSelected(item);
279 }
280
281 private void launchHelpInBrowser() {
282 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
283 try {
284 startActivity(intent);
285 } catch (final ActivityNotFoundException e) {
286 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
287 .show();
288 }
289 }
290
291 private void endCall(View view) {
292 endCall();
293 }
294
295 private void endCall() {
296 if (this.rtpConnectionReference == null) {
297 retractSessionProposal();
298 finish();
299 } else {
300 requireRtpConnection().endCall();
301 }
302 }
303
304 private void retractSessionProposal() {
305 final Intent intent = getIntent();
306 final String action = intent.getAction();
307 final Account account = extractAccount(intent);
308 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
309 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
310 if (!Intent.ACTION_VIEW.equals(action)
311 || state == null
312 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
313 resetIntent(
314 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
315 }
316 xmppConnectionService
317 .getJingleConnectionManager()
318 .retractSessionProposal(account, with.asBareJid());
319 }
320
321 private void rejectCall(View view) {
322 requireRtpConnection().rejectCall();
323 finish();
324 }
325
326 private void acceptCall(View view) {
327 requestPermissionsAndAcceptCall();
328 }
329
330 private void acceptContentAdd() {
331 try {
332 requireRtpConnection()
333 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
334 } catch (final IllegalStateException e) {
335 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
336 }
337 }
338
339 private void requestPermissionAndSwitchToVideo() {
340 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
341 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
342 switchToVideo();
343 }
344 }
345
346 private void switchToVideo() {
347 try {
348 requireRtpConnection().addMedia(Media.VIDEO);
349 } catch (final IllegalStateException e) {
350 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
351 }
352 }
353
354 private void acceptContentAdd(final ContentAddition contentAddition) {
355 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
356 Log.d(Config.LOGTAG,"ignore press on content-accept button");
357 return;
358 }
359 requestPermissionAndAcceptContentAdd(contentAddition);
360 }
361
362 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
363 final List<String> permissions = permissions(contentAddition.media());
364 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
365 requireRtpConnection().acceptContentAdd(contentAddition.summary);
366 }
367 }
368
369 private void rejectContentAdd(final View view) {
370 requireRtpConnection().rejectContentAdd();
371 }
372
373 private void requestPermissionsAndAcceptCall() {
374 final List<String> permissions = permissions(getMedia());
375 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
376 putScreenInCallMode();
377 checkRecorderAndAcceptCall();
378 }
379 }
380
381 private List<String> permissions(final Set<Media> media) {
382 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
383 if (media.contains(Media.VIDEO)) {
384 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
385 } else {
386 permissions.add(Manifest.permission.RECORD_AUDIO);
387 }
388 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
389 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
390 }
391 return permissions.build();
392 }
393
394 private void checkRecorderAndAcceptCall() {
395 checkMicrophoneAvailabilityAsync();
396 try {
397 requireRtpConnection().acceptCall();
398 } catch (final IllegalStateException e) {
399 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
400 }
401 }
402
403 private void checkMicrophoneAvailabilityAsync() {
404 new Thread(new MicrophoneAvailabilityCheck(this)).start();
405 }
406
407 private static class MicrophoneAvailabilityCheck implements Runnable {
408
409 private final WeakReference<Activity> activityReference;
410
411 private MicrophoneAvailabilityCheck(final Activity activity) {
412 this.activityReference = new WeakReference<>(activity);
413 }
414
415 @Override
416 public void run() {
417 final long start = SystemClock.elapsedRealtime();
418 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
419 final long stop = SystemClock.elapsedRealtime();
420 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
421 if (isMicrophoneAvailable) {
422 return;
423 }
424 final Activity activity = activityReference.get();
425 if (activity == null) {
426 return;
427 }
428 activity.runOnUiThread(
429 () ->
430 Toast.makeText(
431 activity,
432 R.string.microphone_unavailable,
433 Toast.LENGTH_LONG)
434 .show());
435 }
436 }
437
438 private void putScreenInCallMode() {
439 putScreenInCallMode(requireRtpConnection().getMedia());
440 }
441
442 private void putScreenInCallMode(final Set<Media> media) {
443 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
444 if (Media.audioOnly(media)) {
445 final JingleRtpConnection rtpConnection =
446 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
447 final AppRTCAudioManager audioManager =
448 rtpConnection == null ? null : rtpConnection.getAudioManager();
449 if (audioManager == null
450 || audioManager.getSelectedAudioDevice()
451 == AppRTCAudioManager.AudioDevice.EARPIECE) {
452 acquireProximityWakeLock();
453 }
454 }
455 lockOrientation(media);
456 }
457
458 private void lockOrientation(final Set<Media> media) {
459 if (Media.audioOnly(media)) {
460 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
461 } else {
462 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
463 }
464 }
465
466 @SuppressLint("WakelockTimeout")
467 private void acquireProximityWakeLock() {
468 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
469 if (powerManager == null) {
470 Log.e(Config.LOGTAG, "power manager not available");
471 return;
472 }
473 if (isFinishing()) {
474 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
475 return;
476 }
477 if (this.mProximityWakeLock == null) {
478 this.mProximityWakeLock =
479 powerManager.newWakeLock(
480 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
481 }
482 if (!this.mProximityWakeLock.isHeld()) {
483 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
484 this.mProximityWakeLock.acquire();
485 }
486 }
487
488 private void releaseProximityWakeLock() {
489 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
490 Log.d(Config.LOGTAG, "releasing proximity wake lock");
491 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
492 this.mProximityWakeLock = null;
493 }
494 }
495
496 private void putProximityWakeLockInProperState(
497 final AppRTCAudioManager.AudioDevice audioDevice) {
498 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
499 acquireProximityWakeLock();
500 } else {
501 releaseProximityWakeLock();
502 }
503 }
504
505 @Override
506 protected void refreshUiReal() {}
507
508 @Override
509 public void onNewIntent(final Intent intent) {
510 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
511 super.onNewIntent(intent);
512 setIntent(intent);
513 if (xmppConnectionService == null) {
514 Log.d(
515 Config.LOGTAG,
516 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
517 return;
518 }
519 final Account account = extractAccount(intent);
520 final String action = intent.getAction();
521 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
522 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
523 if (sessionId != null) {
524 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
525 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
526 return;
527 }
528 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
529 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
530 requestPermissionsAndAcceptCall();
531 resetIntent(intent.getExtras());
532 }
533 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
534 proposeJingleRtpSession(account, with, actionToMedia(action));
535 setWith(account.getRoster().getContact(with), null);
536 } else {
537 throw new IllegalStateException("received onNewIntent without sessionId");
538 }
539 }
540
541 @Override
542 void onBackendConnected() {
543 final Intent intent = getIntent();
544 final String action = intent.getAction();
545 final Account account = extractAccount(intent);
546 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
547 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
548 if (sessionId != null) {
549 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
550 return;
551 }
552 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
553 Log.d(Config.LOGTAG, "intent action was accept");
554 requestPermissionsAndAcceptCall();
555 resetIntent(intent.getExtras());
556 }
557 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
558 proposeJingleRtpSession(account, with, actionToMedia(action));
559 setWith(account.getRoster().getContact(with), null);
560 } else if (Intent.ACTION_VIEW.equals(action)) {
561 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
562 final RtpEndUserState state =
563 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
564 if (state != null) {
565 Log.d(Config.LOGTAG, "restored last state from intent extra");
566 updateButtonConfiguration(state);
567 updateVerifiedShield(false);
568 updateStateDisplay(state);
569 updateIncomingCallScreen(state);
570 invalidateOptionsMenu();
571 }
572 setWith(account.getRoster().getContact(with), state);
573 if (xmppConnectionService
574 .getJingleConnectionManager()
575 .fireJingleRtpConnectionStateUpdates()) {
576 return;
577 }
578 if (END_CARD.contains(state)
579 || xmppConnectionService
580 .getJingleConnectionManager()
581 .hasMatchingProposal(account, with)) {
582 return;
583 }
584 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
585 finish();
586 }
587 }
588
589 private void setWith(final RtpEndUserState state) {
590 setWith(getWith(), state);
591 }
592
593 private void setWith(final Contact contact, final RtpEndUserState state) {
594 binding.with.setText(contact.getDisplayName());
595 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
596 .contains(state)) {
597 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
598 binding.withJid.setVisibility(View.VISIBLE);
599 } else {
600 binding.withJid.setVisibility(View.GONE);
601 }
602 }
603
604 private void proposeJingleRtpSession(
605 final Account account, final Jid with, final Set<Media> media) {
606 checkMicrophoneAvailabilityAsync();
607 if (with.isBareJid()) {
608 xmppConnectionService
609 .getJingleConnectionManager()
610 .proposeJingleRtpSession(account, with, media);
611 } else {
612 final String sessionId =
613 xmppConnectionService
614 .getJingleConnectionManager()
615 .initializeRtpSession(account, with, media);
616 initializeActivityWithRunningRtpSession(account, with, sessionId);
617 resetIntent(account, with, sessionId);
618 }
619 putScreenInCallMode(media);
620 }
621
622 @Override
623 public void onRequestPermissionsResult(
624 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
625 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
626 final PermissionUtils.PermissionResult permissionResult =
627 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
628 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
629 if (requestCode == REQUEST_ACCEPT_CALL) {
630 checkRecorderAndAcceptCall();
631 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
632 acceptContentAdd();
633 } else if (requestCode == REQUEST_ADD_CONTENT) {
634 switchToVideo();
635 }
636 } else {
637 @StringRes int res;
638 final String firstDenied =
639 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
640 if (firstDenied == null) {
641 return;
642 }
643 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
644 res = R.string.no_microphone_permission;
645 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
646 res = R.string.no_camera_permission;
647 } else {
648 throw new IllegalStateException("Invalid permission result request");
649 }
650 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
651 .show();
652 }
653 }
654
655 @Override
656 public void onStart() {
657 super.onStart();
658 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
659 this.binding.remoteVideo.setOnAspectRatioChanged(this);
660 }
661
662 @Override
663 public void onStop() {
664 mHandler.removeCallbacks(mTickExecutor);
665 binding.remoteVideo.release();
666 binding.remoteVideo.setOnAspectRatioChanged(null);
667 binding.localVideo.release();
668 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
669 final JingleRtpConnection jingleRtpConnection =
670 weakReference == null ? null : weakReference.get();
671 if (jingleRtpConnection != null) {
672 releaseVideoTracks(jingleRtpConnection);
673 }
674 releaseProximityWakeLock();
675 super.onStop();
676 }
677
678 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
679 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
680 if (remoteVideo.isPresent()) {
681 remoteVideo.get().removeSink(binding.remoteVideo);
682 }
683 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
684 if (localVideo.isPresent()) {
685 localVideo.get().removeSink(binding.localVideo);
686 }
687 }
688
689 @Override
690 public void onBackPressed() {
691 if (isConnected()) {
692 if (switchToPictureInPicture()) {
693 return;
694 }
695 } else {
696 endCall();
697 }
698 super.onBackPressed();
699 }
700
701 @Override
702 public void onUserLeaveHint() {
703 super.onUserLeaveHint();
704 if (switchToPictureInPicture()) {
705 return;
706 }
707 // TODO apparently this method is not getting called on Android 10 when using the task
708 // switcher
709 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
710 retractSessionProposal();
711 }
712 }
713
714 private boolean isConnected() {
715 final JingleRtpConnection connection =
716 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
717 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
718 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
719 }
720
721 private boolean switchToPictureInPicture() {
722 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
723 if (shouldBePictureInPicture()) {
724 startPictureInPicture();
725 return true;
726 }
727 }
728 return false;
729 }
730
731 @RequiresApi(api = Build.VERSION_CODES.O)
732 private void startPictureInPicture() {
733 try {
734 final Rational rational = this.binding.remoteVideo.getAspectRatio();
735 final Rational clippedRational = Rationals.clip(rational);
736 Log.d(
737 Config.LOGTAG,
738 "suggested rational " + rational + ". clipped to " + clippedRational);
739 enterPictureInPictureMode(
740 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
741 } catch (final IllegalStateException e) {
742 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
743 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
744 }
745 }
746
747 @Override
748 public void onAspectRatioChanged(final Rational rational) {
749 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
750 final Rational clippedRational = Rationals.clip(rational);
751 Log.d(
752 Config.LOGTAG,
753 "suggested rational after aspect ratio change "
754 + rational
755 + ". clipped to "
756 + clippedRational);
757 setPictureInPictureParams(
758 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
759 }
760 }
761
762 private boolean deviceSupportsPictureInPicture() {
763 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
764 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
765 } else {
766 return false;
767 }
768 }
769
770 private boolean shouldBePictureInPicture() {
771 try {
772 final JingleRtpConnection rtpConnection = requireRtpConnection();
773 return rtpConnection.getMedia().contains(Media.VIDEO)
774 && Arrays.asList(
775 RtpEndUserState.ACCEPTING_CALL,
776 RtpEndUserState.CONNECTING,
777 RtpEndUserState.CONNECTED)
778 .contains(rtpConnection.getEndUserState());
779 } catch (final IllegalStateException e) {
780 return false;
781 }
782 }
783
784 private boolean initializeActivityWithRunningRtpSession(
785 final Account account, Jid with, String sessionId) {
786 final WeakReference<JingleRtpConnection> reference =
787 xmppConnectionService
788 .getJingleConnectionManager()
789 .findJingleRtpConnection(account, with, sessionId);
790 if (reference == null || reference.get() == null) {
791 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
792 xmppConnectionService
793 .getJingleConnectionManager()
794 .getTerminalSessionState(with, sessionId);
795 if (terminatedRtpSession == null) {
796 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
797 finish();
798 return true;
799 }
800 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
801 return true;
802 }
803 this.rtpConnectionReference = reference;
804 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
805 final boolean verified = requireRtpConnection().isVerified();
806 if (currentState == RtpEndUserState.ENDED) {
807 finish();
808 return true;
809 }
810 final Set<Media> media = getMedia();
811 final ContentAddition contentAddition = getPendingContentAddition();
812 if (currentState == RtpEndUserState.INCOMING_CALL) {
813 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
814 }
815 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
816 requireRtpConnection().getState())) {
817 putScreenInCallMode();
818 }
819 setWith(currentState);
820 updateVideoViews(currentState);
821 updateStateDisplay(currentState, media, contentAddition);
822 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
823 updateButtonConfiguration(currentState, media, contentAddition);
824 updateIncomingCallScreen(currentState);
825 invalidateOptionsMenu();
826 return false;
827 }
828
829 private void initializeWithTerminatedSessionState(
830 final Account account,
831 final Jid with,
832 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
833 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
834 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
835 finish();
836 return;
837 }
838 final RtpEndUserState state = terminatedRtpSession.state;
839 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
840 updateButtonConfiguration(state);
841 updateStateDisplay(state);
842 updateIncomingCallScreen(state);
843 updateCallDuration();
844 updateVerifiedShield(false);
845 invalidateOptionsMenu();
846 setWith(account.getRoster().getContact(with), state);
847 }
848
849 private void reInitializeActivityWithRunningRtpSession(
850 final Account account, Jid with, String sessionId) {
851 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
852 resetIntent(account, with, sessionId);
853 }
854
855 private void resetIntent(final Account account, final Jid with, final String sessionId) {
856 final Intent intent = new Intent(Intent.ACTION_VIEW);
857 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
858 intent.putExtra(EXTRA_WITH, with.toEscapedString());
859 intent.putExtra(EXTRA_SESSION_ID, sessionId);
860 setIntent(intent);
861 }
862
863 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
864 surfaceViewRenderer.setVisibility(View.VISIBLE);
865 try {
866 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
867 } catch (final IllegalStateException e) {
868 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
869 }
870 surfaceViewRenderer.setEnableHardwareScaler(true);
871 }
872
873 private void updateStateDisplay(final RtpEndUserState state) {
874 updateStateDisplay(state, Collections.emptySet(), null);
875 }
876
877 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
878 switch (state) {
879 case INCOMING_CALL:
880 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
881 if (media.contains(Media.VIDEO)) {
882 setTitle(R.string.rtp_state_incoming_video_call);
883 } else {
884 setTitle(R.string.rtp_state_incoming_call);
885 }
886 break;
887 case INCOMING_CONTENT_ADD:
888 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
889 setTitle(R.string.rtp_state_content_add_video);
890 } else {
891 setTitle(R.string.rtp_state_content_add);
892 }
893 break;
894 case CONNECTING:
895 setTitle(R.string.rtp_state_connecting);
896 break;
897 case CONNECTED:
898 setTitle(R.string.rtp_state_connected);
899 break;
900 case RECONNECTING:
901 setTitle(R.string.rtp_state_reconnecting);
902 break;
903 case ACCEPTING_CALL:
904 setTitle(R.string.rtp_state_accepting_call);
905 break;
906 case ENDING_CALL:
907 setTitle(R.string.rtp_state_ending_call);
908 break;
909 case FINDING_DEVICE:
910 setTitle(R.string.rtp_state_finding_device);
911 break;
912 case RINGING:
913 setTitle(R.string.rtp_state_ringing);
914 break;
915 case DECLINED_OR_BUSY:
916 setTitle(R.string.rtp_state_declined_or_busy);
917 break;
918 case CONNECTIVITY_ERROR:
919 setTitle(R.string.rtp_state_connectivity_error);
920 break;
921 case CONNECTIVITY_LOST_ERROR:
922 setTitle(R.string.rtp_state_connectivity_lost_error);
923 break;
924 case RETRACTED:
925 setTitle(R.string.rtp_state_retracted);
926 break;
927 case APPLICATION_ERROR:
928 setTitle(R.string.rtp_state_application_failure);
929 break;
930 case SECURITY_ERROR:
931 setTitle(R.string.rtp_state_security_error);
932 break;
933 case ENDED:
934 throw new IllegalStateException(
935 "Activity should have called finishAndReleaseWakeLock();");
936 default:
937 throw new IllegalStateException(
938 String.format("State %s has not been handled in UI", state));
939 }
940 }
941
942 private void updateVerifiedShield(final boolean verified) {
943 if (isPictureInPicture()) {
944 this.binding.verified.setVisibility(View.GONE);
945 return;
946 }
947 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
948 }
949
950 private void updateIncomingCallScreen(final RtpEndUserState state) {
951 updateIncomingCallScreen(state, null);
952 }
953
954 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
955 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
956 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
957 if (show) {
958 binding.contactPhoto.setVisibility(View.VISIBLE);
959 if (contact == null) {
960 AvatarWorkerTask.loadAvatar(
961 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
962 } else {
963 AvatarWorkerTask.loadAvatar(
964 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
965 }
966 } else {
967 binding.contactPhoto.setVisibility(View.GONE);
968 }
969 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
970 binding.usingAccount.setVisibility(View.VISIBLE);
971 binding.usingAccount.setText(
972 getString(
973 R.string.using_account,
974 account.getJid().asBareJid().toEscapedString()));
975 } else {
976 binding.usingAccount.setVisibility(View.GONE);
977 binding.contactPhoto.setVisibility(View.GONE);
978 }
979 }
980
981 private Set<Media> getMedia() {
982 return requireRtpConnection().getMedia();
983 }
984
985 public ContentAddition getPendingContentAddition() {
986 return requireRtpConnection().getPendingContentAddition();
987 }
988
989 private void updateButtonConfiguration(final RtpEndUserState state) {
990 updateButtonConfiguration(state, Collections.emptySet(), null);
991 }
992
993 @SuppressLint("RestrictedApi")
994 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
995 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
996 this.binding.rejectCall.setVisibility(View.INVISIBLE);
997 this.binding.endCall.setVisibility(View.INVISIBLE);
998 this.binding.acceptCall.setVisibility(View.INVISIBLE);
999 } else if (state == RtpEndUserState.INCOMING_CALL) {
1000 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
1001 this.binding.rejectCall.setOnClickListener(this::rejectCall);
1002 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1003 this.binding.rejectCall.setVisibility(View.VISIBLE);
1004 this.binding.endCall.setVisibility(View.INVISIBLE);
1005 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
1006 this.binding.acceptCall.setOnClickListener(this::acceptCall);
1007 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
1008 this.binding.acceptCall.setVisibility(View.VISIBLE);
1009 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
1010 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
1011 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
1012 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1013 this.binding.rejectCall.setVisibility(View.VISIBLE);
1014 this.binding.endCall.setVisibility(View.INVISIBLE);
1015 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1016 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1017 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
1018 this.binding.acceptCall.setVisibility(View.VISIBLE);
1019 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
1020 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1021 this.binding.rejectCall.setOnClickListener(this::exit);
1022 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1023 this.binding.rejectCall.setVisibility(View.VISIBLE);
1024 this.binding.endCall.setVisibility(View.INVISIBLE);
1025 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1026 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1027 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1028 this.binding.acceptCall.setVisibility(View.VISIBLE);
1029 } else if (asList(
1030 RtpEndUserState.CONNECTIVITY_ERROR,
1031 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1032 RtpEndUserState.APPLICATION_ERROR,
1033 RtpEndUserState.RETRACTED,
1034 RtpEndUserState.SECURITY_ERROR)
1035 .contains(state)) {
1036 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1037 this.binding.rejectCall.setOnClickListener(this::exit);
1038 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1039 this.binding.rejectCall.setVisibility(View.VISIBLE);
1040 this.binding.endCall.setVisibility(View.INVISIBLE);
1041 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1042 this.binding.acceptCall.setOnClickListener(this::retry);
1043 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1044 this.binding.acceptCall.setVisibility(View.VISIBLE);
1045 } else {
1046 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1047 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1048 this.binding.endCall.setOnClickListener(this::endCall);
1049 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1050 this.binding.endCall.setVisibility(View.VISIBLE);
1051 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1052 }
1053 updateInCallButtonConfiguration(state, media);
1054 }
1055
1056 private boolean isPictureInPicture() {
1057 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1058 return isInPictureInPictureMode();
1059 } else {
1060 return false;
1061 }
1062 }
1063
1064 private void updateInCallButtonConfiguration() {
1065 updateInCallButtonConfiguration(
1066 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1067 }
1068
1069 @SuppressLint("RestrictedApi")
1070 private void updateInCallButtonConfiguration(
1071 final RtpEndUserState state, final Set<Media> media) {
1072 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1073 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1074 if (media.contains(Media.VIDEO)) {
1075 final JingleRtpConnection rtpConnection = requireRtpConnection();
1076 updateInCallButtonConfigurationVideo(
1077 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1078 } else {
1079 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1080 updateInCallButtonConfigurationSpeaker(
1081 audioManager.getSelectedAudioDevice(),
1082 audioManager.getAudioDevices().size());
1083 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1084 }
1085 if (media.contains(Media.AUDIO)) {
1086 updateInCallButtonConfigurationMicrophone(
1087 requireRtpConnection().isMicrophoneEnabled());
1088 } else {
1089 this.binding.inCallActionLeft.setVisibility(View.GONE);
1090 }
1091 } else {
1092 this.binding.inCallActionLeft.setVisibility(View.GONE);
1093 this.binding.inCallActionRight.setVisibility(View.GONE);
1094 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1095 }
1096 }
1097
1098 @SuppressLint("RestrictedApi")
1099 private void updateInCallButtonConfigurationSpeaker(
1100 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1101 switch (selectedAudioDevice) {
1102 case EARPIECE:
1103 this.binding.inCallActionRight.setImageResource(
1104 R.drawable.ic_volume_off_black_24dp);
1105 if (numberOfChoices >= 2) {
1106 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1107 } else {
1108 this.binding.inCallActionRight.setOnClickListener(null);
1109 this.binding.inCallActionRight.setClickable(false);
1110 }
1111 break;
1112 case WIRED_HEADSET:
1113 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1114 this.binding.inCallActionRight.setOnClickListener(null);
1115 this.binding.inCallActionRight.setClickable(false);
1116 break;
1117 case SPEAKER_PHONE:
1118 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1119 if (numberOfChoices >= 2) {
1120 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1121 } else {
1122 this.binding.inCallActionRight.setOnClickListener(null);
1123 this.binding.inCallActionRight.setClickable(false);
1124 }
1125 break;
1126 case BLUETOOTH:
1127 this.binding.inCallActionRight.setImageResource(
1128 R.drawable.ic_bluetooth_audio_black_24dp);
1129 this.binding.inCallActionRight.setOnClickListener(null);
1130 this.binding.inCallActionRight.setClickable(false);
1131 break;
1132 }
1133 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1134 }
1135
1136 @SuppressLint("RestrictedApi")
1137 private void updateInCallButtonConfigurationVideo(
1138 final boolean videoEnabled, final boolean isCameraSwitchable) {
1139 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1140 if (isCameraSwitchable) {
1141 this.binding.inCallActionFarRight.setImageResource(
1142 R.drawable.ic_flip_camera_android_black_24dp);
1143 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1144 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1145 } else {
1146 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1147 }
1148 if (videoEnabled) {
1149 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1150 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1151 } else {
1152 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1153 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1154 }
1155 }
1156
1157 private void switchCamera(final View view) {
1158 Futures.addCallback(
1159 requireRtpConnection().switchCamera(),
1160 new FutureCallback<Boolean>() {
1161 @Override
1162 public void onSuccess(@Nullable Boolean isFrontCamera) {
1163 binding.localVideo.setMirror(isFrontCamera);
1164 }
1165
1166 @Override
1167 public void onFailure(@NonNull final Throwable throwable) {
1168 Log.d(
1169 Config.LOGTAG,
1170 "could not switch camera",
1171 Throwables.getRootCause(throwable));
1172 Toast.makeText(
1173 RtpSessionActivity.this,
1174 R.string.could_not_switch_camera,
1175 Toast.LENGTH_LONG)
1176 .show();
1177 }
1178 },
1179 MainThreadExecutor.getInstance());
1180 }
1181
1182 private void enableVideo(View view) {
1183 try {
1184 requireRtpConnection().setVideoEnabled(true);
1185 } catch (final IllegalStateException e) {
1186 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1187 return;
1188 }
1189 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1190 }
1191
1192 private void disableVideo(View view) {
1193 final JingleRtpConnection rtpConnection = requireRtpConnection();
1194 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1195 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1196 rtpConnection.retractContentAdd();
1197 return;
1198 }
1199 requireRtpConnection().setVideoEnabled(false);
1200 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1201 }
1202
1203 @SuppressLint("RestrictedApi")
1204 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1205 if (microphoneEnabled) {
1206 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1207 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1208 } else {
1209 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1210 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1211 }
1212 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1213 }
1214
1215 private void updateCallDuration() {
1216 final JingleRtpConnection connection =
1217 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1218 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1219 this.binding.duration.setVisibility(View.GONE);
1220 return;
1221 }
1222 if (connection.zeroDuration()) {
1223 this.binding.duration.setVisibility(View.GONE);
1224 } else {
1225 this.binding.duration.setText(
1226 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1227 this.binding.duration.setVisibility(View.VISIBLE);
1228 }
1229 }
1230
1231 private void updateVideoViews(final RtpEndUserState state) {
1232 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1233 binding.localVideo.setVisibility(View.GONE);
1234 binding.localVideo.release();
1235 binding.remoteVideoWrapper.setVisibility(View.GONE);
1236 binding.remoteVideo.release();
1237 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1238 if (isPictureInPicture()) {
1239 binding.appBarLayout.setVisibility(View.GONE);
1240 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1241 if (Arrays.asList(
1242 RtpEndUserState.APPLICATION_ERROR,
1243 RtpEndUserState.CONNECTIVITY_ERROR,
1244 RtpEndUserState.SECURITY_ERROR)
1245 .contains(state)) {
1246 binding.pipWarning.setVisibility(View.VISIBLE);
1247 binding.pipWaiting.setVisibility(View.GONE);
1248 } else {
1249 binding.pipWarning.setVisibility(View.GONE);
1250 binding.pipWaiting.setVisibility(View.GONE);
1251 }
1252 } else {
1253 binding.appBarLayout.setVisibility(View.VISIBLE);
1254 binding.pipPlaceholder.setVisibility(View.GONE);
1255 }
1256 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1257 return;
1258 }
1259 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1260 binding.localVideo.setVisibility(View.GONE);
1261 binding.remoteVideoWrapper.setVisibility(View.GONE);
1262 binding.appBarLayout.setVisibility(View.GONE);
1263 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1264 binding.pipWarning.setVisibility(View.GONE);
1265 binding.pipWaiting.setVisibility(View.VISIBLE);
1266 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1267 return;
1268 }
1269 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1270 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1271 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1272 // paint local view over remote view
1273 binding.localVideo.setZOrderMediaOverlay(true);
1274 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1275 addSink(localVideoTrack.get(), binding.localVideo);
1276 } else {
1277 binding.localVideo.setVisibility(View.GONE);
1278 }
1279 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1280 if (remoteVideoTrack.isPresent()) {
1281 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1282 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1283 binding.remoteVideo.setScalingType(
1284 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1285 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1286 if (state == RtpEndUserState.CONNECTED) {
1287 binding.appBarLayout.setVisibility(View.GONE);
1288 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1289 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1290 } else {
1291 binding.appBarLayout.setVisibility(View.VISIBLE);
1292 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1293 binding.remoteVideoWrapper.setVisibility(View.GONE);
1294 }
1295 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1296 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1297 } else {
1298 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1299 }
1300 } else {
1301 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1302 binding.remoteVideoWrapper.setVisibility(View.GONE);
1303 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1304 }
1305 }
1306
1307 private Optional<VideoTrack> getLocalVideoTrack() {
1308 final JingleRtpConnection connection =
1309 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1310 if (connection == null) {
1311 return Optional.absent();
1312 }
1313 return connection.getLocalVideoTrack();
1314 }
1315
1316 private Optional<VideoTrack> getRemoteVideoTrack() {
1317 final JingleRtpConnection connection =
1318 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1319 if (connection == null) {
1320 return Optional.absent();
1321 }
1322 return connection.getRemoteVideoTrack();
1323 }
1324
1325 private void disableMicrophone(View view) {
1326 final JingleRtpConnection rtpConnection = requireRtpConnection();
1327 if (rtpConnection.setMicrophoneEnabled(false)) {
1328 updateInCallButtonConfiguration();
1329 }
1330 }
1331
1332 private void enableMicrophone(View view) {
1333 final JingleRtpConnection rtpConnection = requireRtpConnection();
1334 if (rtpConnection.setMicrophoneEnabled(true)) {
1335 updateInCallButtonConfiguration();
1336 }
1337 }
1338
1339 private void switchToEarpiece(View view) {
1340 requireRtpConnection()
1341 .getAudioManager()
1342 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1343 acquireProximityWakeLock();
1344 }
1345
1346 private void switchToSpeaker(View view) {
1347 requireRtpConnection()
1348 .getAudioManager()
1349 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1350 releaseProximityWakeLock();
1351 }
1352
1353 private void retry(View view) {
1354 final Intent intent = getIntent();
1355 final Account account = extractAccount(intent);
1356 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1357 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1358 final String action = intent.getAction();
1359 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1360 this.rtpConnectionReference = null;
1361 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1362 proposeJingleRtpSession(account, with, media);
1363 }
1364
1365 private void exit(final View view) {
1366 finish();
1367 }
1368
1369 private void recordVoiceMail(final View view) {
1370 final Intent intent = getIntent();
1371 final Account account = extractAccount(intent);
1372 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1373 final Conversation conversation =
1374 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1375 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1376 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1377 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1378 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1379 launchIntent.putExtra(
1380 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1381 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1382 startActivity(launchIntent);
1383 finish();
1384 }
1385
1386 private Contact getWith() {
1387 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1388 final Account account = id.account;
1389 return account.getRoster().getContact(id.with);
1390 }
1391
1392 private JingleRtpConnection requireRtpConnection() {
1393 final JingleRtpConnection connection =
1394 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1395 if (connection == null) {
1396 throw new IllegalStateException("No RTP connection found");
1397 }
1398 return connection;
1399 }
1400
1401 @Override
1402 public void onJingleRtpConnectionUpdate(
1403 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1404 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1405 if (END_CARD.contains(state)) {
1406 Log.d(Config.LOGTAG, "end card reached");
1407 releaseProximityWakeLock();
1408 runOnUiThread(
1409 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1410 }
1411 if (with.isBareJid()) {
1412 updateRtpSessionProposalState(account, with, state);
1413 return;
1414 }
1415 if (emptyReference(this.rtpConnectionReference)) {
1416 if (END_CARD.contains(state)) {
1417 Log.d(Config.LOGTAG, "not reinitializing session");
1418 return;
1419 }
1420 // this happens when going from proposed session to actual session
1421 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1422 return;
1423 }
1424 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1425 final boolean verified = requireRtpConnection().isVerified();
1426 final Set<Media> media = getMedia();
1427 lockOrientation(media);
1428 final ContentAddition contentAddition = getPendingContentAddition();
1429 final Contact contact = getWith();
1430 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1431 if (state == RtpEndUserState.ENDED) {
1432 finish();
1433 return;
1434 }
1435 runOnUiThread(
1436 () -> {
1437 updateStateDisplay(state, media, contentAddition);
1438 updateVerifiedShield(
1439 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1440 updateButtonConfiguration(state, media, contentAddition);
1441 updateVideoViews(state);
1442 updateIncomingCallScreen(state, contact);
1443 invalidateOptionsMenu();
1444 });
1445 if (END_CARD.contains(state)) {
1446 final JingleRtpConnection rtpConnection = requireRtpConnection();
1447 resetIntent(account, with, state, rtpConnection.getMedia());
1448 releaseVideoTracks(rtpConnection);
1449 this.rtpConnectionReference = null;
1450 }
1451 } else {
1452 Log.d(Config.LOGTAG, "received update for other rtp session");
1453 }
1454 }
1455
1456 @Override
1457 public void onAudioDeviceChanged(
1458 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1459 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1460 Log.d(
1461 Config.LOGTAG,
1462 "onAudioDeviceChanged in activity: selected:"
1463 + selectedAudioDevice
1464 + ", available:"
1465 + availableAudioDevices);
1466 try {
1467 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1468 final Set<Media> media = getMedia();
1469 if (END_CARD.contains(endUserState)) {
1470 Log.d(
1471 Config.LOGTAG,
1472 "onAudioDeviceChanged() nothing to do because end card has been reached");
1473 } else {
1474 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1475 final AppRTCAudioManager audioManager =
1476 requireRtpConnection().getAudioManager();
1477 updateInCallButtonConfigurationSpeaker(
1478 audioManager.getSelectedAudioDevice(),
1479 audioManager.getAudioDevices().size());
1480 }
1481 Log.d(
1482 Config.LOGTAG,
1483 "put proximity wake lock into proper state after device update");
1484 putProximityWakeLockInProperState(selectedAudioDevice);
1485 }
1486 } catch (final IllegalStateException e) {
1487 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1488 }
1489 }
1490
1491 @Override
1492 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1493 super.onSaveInstanceState(outState);
1494 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1495 }
1496
1497 private void updateRtpSessionProposalState(
1498 final Account account, final Jid with, final RtpEndUserState state) {
1499 final Intent currentIntent = getIntent();
1500 final String withExtra =
1501 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1502 if (withExtra == null) {
1503 return;
1504 }
1505 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1506 runOnUiThread(
1507 () -> {
1508 updateVerifiedShield(false);
1509 updateStateDisplay(state);
1510 updateButtonConfiguration(state);
1511 updateIncomingCallScreen(state);
1512 invalidateOptionsMenu();
1513 });
1514 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1515 }
1516 }
1517
1518 private void resetIntent(final Bundle extras) {
1519 final Intent intent = new Intent(Intent.ACTION_VIEW);
1520 intent.putExtras(extras);
1521 setIntent(intent);
1522 }
1523
1524 private void resetIntent(
1525 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1526 final Intent intent = new Intent(Intent.ACTION_VIEW);
1527 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1528 if (account.getRoster()
1529 .getContact(with)
1530 .getPresences()
1531 .anySupport(Namespace.JINGLE_MESSAGE)) {
1532 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1533 } else {
1534 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1535 }
1536 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1537 intent.putExtra(
1538 EXTRA_LAST_ACTION,
1539 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1540 setIntent(intent);
1541 }
1542
1543 private static boolean emptyReference(final WeakReference<?> weakReference) {
1544 return weakReference == null || weakReference.get() == null;
1545 }
1546}