1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.os.SystemClock;
21import android.util.Log;
22import android.util.Rational;
23import android.view.KeyEvent;
24import android.view.Menu;
25import android.view.MenuItem;
26import android.view.View;
27import android.view.WindowManager;
28import android.widget.Toast;
29
30import androidx.annotation.NonNull;
31import androidx.annotation.Nullable;
32import androidx.annotation.RequiresApi;
33import androidx.annotation.StringRes;
34import androidx.databinding.DataBindingUtil;
35
36import com.google.common.base.Optional;
37import com.google.common.base.Preconditions;
38import com.google.common.base.Throwables;
39import com.google.common.collect.ImmutableList;
40import com.google.common.collect.ImmutableSet;
41import com.google.common.util.concurrent.FutureCallback;
42import com.google.common.util.concurrent.Futures;
43
44import org.jetbrains.annotations.NotNull;
45import org.webrtc.RendererCommon;
46import org.webrtc.SurfaceViewRenderer;
47import org.webrtc.VideoTrack;
48
49import java.lang.ref.WeakReference;
50import java.util.Arrays;
51import java.util.Collections;
52import java.util.List;
53import java.util.Set;
54
55import eu.siacs.conversations.Config;
56import eu.siacs.conversations.R;
57import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
58import eu.siacs.conversations.entities.Account;
59import eu.siacs.conversations.entities.Contact;
60import eu.siacs.conversations.entities.Conversation;
61import eu.siacs.conversations.services.AppRTCAudioManager;
62import eu.siacs.conversations.services.XmppConnectionService;
63import eu.siacs.conversations.ui.widget.DialpadView;
64import eu.siacs.conversations.ui.util.AvatarWorkerTask;
65import eu.siacs.conversations.ui.util.MainThreadExecutor;
66import eu.siacs.conversations.ui.util.Rationals;
67import eu.siacs.conversations.utils.PermissionUtils;
68import eu.siacs.conversations.utils.TimeFrameUtils;
69import eu.siacs.conversations.xmpp.Jid;
70import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
71import eu.siacs.conversations.xmpp.jingle.ContentAddition;
72import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
73import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
74import eu.siacs.conversations.xmpp.jingle.Media;
75import eu.siacs.conversations.xmpp.jingle.RtpCapability;
76import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
77
78import org.webrtc.RendererCommon;
79import org.webrtc.SurfaceViewRenderer;
80import org.webrtc.VideoTrack;
81
82import java.lang.ref.WeakReference;
83import java.util.Arrays;
84import java.util.Collections;
85import java.util.List;
86import java.util.Set;
87
88public class RtpSessionActivity extends XmppActivity
89 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
90 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
91
92 public static final String EXTRA_WITH = "with";
93 public static final String EXTRA_SESSION_ID = "session_id";
94 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
95 public static final String EXTRA_LAST_ACTION = "last_action";
96 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
97 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
98 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
99
100 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
101
102 public static final List<RtpEndUserState> END_CARD =
103 Arrays.asList(
104 RtpEndUserState.APPLICATION_ERROR,
105 RtpEndUserState.SECURITY_ERROR,
106 RtpEndUserState.DECLINED_OR_BUSY,
107 RtpEndUserState.CONNECTIVITY_ERROR,
108 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
109 RtpEndUserState.RETRACTED);
110 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
111 Arrays.asList(
112 RtpEndUserState.APPLICATION_ERROR,
113 RtpEndUserState.CONNECTIVITY_ERROR,
114 RtpEndUserState.SECURITY_ERROR);
115 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
116 Arrays.asList(
117 RtpEndUserState.CONNECTING,
118 RtpEndUserState.CONNECTED,
119 RtpEndUserState.RECONNECTING,
120 RtpEndUserState.INCOMING_CONTENT_ADD);
121 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
122 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
123 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
124 Arrays.asList(
125 RtpEndUserState.ACCEPTING_CALL,
126 RtpEndUserState.CONNECTING,
127 RtpEndUserState.RECONNECTING);
128 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
129 private static final int REQUEST_ACCEPT_CALL = 0x1111;
130 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
131 private static final int REQUEST_ADD_CONTENT = 0x1113;
132 private WeakReference<JingleRtpConnection> rtpConnectionReference;
133
134 private ActivityRtpSessionBinding binding;
135 private PowerManager.WakeLock mProximityWakeLock;
136
137 private final Handler mHandler = new Handler();
138 private final Runnable mTickExecutor =
139 new Runnable() {
140 @Override
141 public void run() {
142 updateCallDuration();
143 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
144 }
145 };
146
147 private static Set<Media> actionToMedia(final String action) {
148 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
149 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
150 } else {
151 return ImmutableSet.of(Media.AUDIO);
152 }
153 }
154
155 private static void addSink(
156 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
157 try {
158 videoTrack.addSink(surfaceViewRenderer);
159 } catch (final IllegalStateException e) {
160 Log.e(
161 Config.LOGTAG,
162 "possible race condition on trying to display video track. ignoring",
163 e);
164 }
165 }
166
167 @Override
168 public void onCreate(Bundle savedInstanceState) {
169 super.onCreate(savedInstanceState);
170 getWindow()
171 .addFlags(
172 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
173 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
174 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
175 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
176 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
177 setSupportActionBar(binding.toolbar);
178
179 binding.dialpad.setClickConsumer(tag -> {
180 final JingleRtpConnection connection =
181 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
182 if (connection != null) connection.applyDtmfTone(tag);
183 });
184
185 if (savedInstanceState != null) {
186 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
187 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
188 }
189 }
190
191 @Override
192 public boolean onCreateOptionsMenu(final Menu menu) {
193 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
194 final MenuItem help = menu.findItem(R.id.action_help);
195 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
196 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
197 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
198 help.setVisible(Config.HELP != null && isHelpButtonVisible());
199 gotoChat.setVisible(isSwitchToConversationVisible());
200 switchToVideo.setVisible(isSwitchToVideoVisible());
201 dialpad.setVisible(isAudioOnlyConversation());
202 return super.onCreateOptionsMenu(menu);
203 }
204
205 @Override
206 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
207 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
208 if (xmppConnectionService != null) {
209 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
210 return true;
211 }
212 }
213 }
214 return super.onKeyDown(keyCode, event);
215 }
216
217 private boolean isHelpButtonVisible() {
218 try {
219 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
220 } catch (IllegalStateException e) {
221 final Intent intent = getIntent();
222 final String state =
223 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
224 if (state != null) {
225 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
226 } else {
227 return false;
228 }
229 }
230 }
231
232 private boolean isSwitchToConversationVisible() {
233 final JingleRtpConnection connection =
234 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
235 return connection != null
236 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
237 }
238
239 private boolean isAudioOnlyConversation() {
240 final JingleRtpConnection connection =
241 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
242
243 return connection != null && !connection.getMedia().contains(Media.VIDEO);
244 }
245
246 private boolean isSwitchToVideoVisible() {
247 final JingleRtpConnection connection =
248 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
249 if (connection == null) {
250 return false;
251 }
252 return connection.isSwitchToVideoAvailable();
253 }
254
255 private void switchToConversation() {
256 final Contact contact = getWith();
257 final Conversation conversation =
258 xmppConnectionService.findOrCreateConversation(
259 contact.getAccount(), contact.getJid(), false, true);
260 switchToConversation(conversation);
261 }
262
263 private void toggleDialpadVisibility() {
264 if (binding.dialpad.getVisibility() == View.VISIBLE) {
265 binding.dialpad.setVisibility(View.GONE);
266 }
267 else {
268 binding.dialpad.setVisibility(View.VISIBLE);
269 }
270 }
271
272 public boolean onOptionsItemSelected(final MenuItem item) {
273 final var itemItem = item.getItemId();
274 if (itemItem == R.id.action_help) {
275 launchHelpInBrowser();
276 return true;
277 } else if (itemItem == R.id.action_goto_chat) {
278 switchToConversation();
279 return true;
280 } else if (itemItem == R.id.action_dialpad) {
281 toggleDialpadVisibility();
282 return true;
283 } else if (itemItem == R.id.action_switch_to_video) {
284 requestPermissionAndSwitchToVideo();
285 return true;
286 } else {
287 return super.onOptionsItemSelected(item);
288 }
289 }
290
291 private void launchHelpInBrowser() {
292 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
293 try {
294 startActivity(intent);
295 } catch (final ActivityNotFoundException e) {
296 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
297 .show();
298 }
299 }
300
301 private void endCall(View view) {
302 endCall();
303 }
304
305 private void endCall() {
306 if (this.rtpConnectionReference == null) {
307 retractSessionProposal();
308 finish();
309 } else {
310 requireRtpConnection().endCall();
311 }
312 }
313
314 private void retractSessionProposal() {
315 final Intent intent = getIntent();
316 final String action = intent.getAction();
317 final Account account = extractAccount(intent);
318 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
319 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
320 if (!Intent.ACTION_VIEW.equals(action)
321 || state == null
322 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
323 resetIntent(
324 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
325 }
326 xmppConnectionService
327 .getJingleConnectionManager()
328 .retractSessionProposal(account, with.asBareJid());
329 }
330
331 private void rejectCall(View view) {
332 requireRtpConnection().rejectCall();
333 finish();
334 }
335
336 private void acceptCall(View view) {
337 requestPermissionsAndAcceptCall();
338 }
339
340 private void acceptContentAdd() {
341 try {
342 requireRtpConnection()
343 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
344 } catch (final IllegalStateException e) {
345 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
346 }
347 }
348
349 private void requestPermissionAndSwitchToVideo() {
350 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
351 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
352 switchToVideo();
353 }
354 }
355
356 private void switchToVideo() {
357 try {
358 requireRtpConnection().addMedia(Media.VIDEO);
359 } catch (final IllegalStateException e) {
360 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
361 }
362 }
363
364 private void acceptContentAdd(final ContentAddition contentAddition) {
365 if (contentAddition == null
366 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
367 Log.d(Config.LOGTAG, "ignore press on content-accept button");
368 return;
369 }
370 requestPermissionAndAcceptContentAdd(contentAddition);
371 }
372
373 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
374 final List<String> permissions = permissions(contentAddition.media());
375 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
376 try {
377 requireRtpConnection().acceptContentAdd(contentAddition.summary);
378 } catch (final IllegalStateException e) {
379 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
380 }
381 }
382 }
383
384 private void rejectContentAdd(final View view) {
385 requireRtpConnection().rejectContentAdd();
386 }
387
388 private void requestPermissionsAndAcceptCall() {
389 final List<String> permissions = permissions(getMedia());
390 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
391 putScreenInCallMode();
392 checkRecorderAndAcceptCall();
393 }
394 }
395
396 private List<String> permissions(final Set<Media> media) {
397 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
398 if (media.contains(Media.VIDEO)) {
399 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
400 } else {
401 permissions.add(Manifest.permission.RECORD_AUDIO);
402 }
403 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
404 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
405 }
406 return permissions.build();
407 }
408
409 private void checkRecorderAndAcceptCall() {
410 checkMicrophoneAvailabilityAsync();
411 try {
412 requireRtpConnection().acceptCall();
413 } catch (final IllegalStateException e) {
414 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
415 }
416 }
417
418 private void checkMicrophoneAvailabilityAsync() {
419 new Thread(new MicrophoneAvailabilityCheck(this)).start();
420 }
421
422 private static class MicrophoneAvailabilityCheck implements Runnable {
423
424 private final WeakReference<Activity> activityReference;
425
426 private MicrophoneAvailabilityCheck(final Activity activity) {
427 this.activityReference = new WeakReference<>(activity);
428 }
429
430 @Override
431 public void run() {
432 final long start = SystemClock.elapsedRealtime();
433 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
434 final long stop = SystemClock.elapsedRealtime();
435 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
436 if (isMicrophoneAvailable) {
437 return;
438 }
439 final Activity activity = activityReference.get();
440 if (activity == null) {
441 return;
442 }
443 activity.runOnUiThread(
444 () ->
445 Toast.makeText(
446 activity,
447 R.string.microphone_unavailable,
448 Toast.LENGTH_LONG)
449 .show());
450 }
451 }
452
453 private void putScreenInCallMode() {
454 putScreenInCallMode(requireRtpConnection().getMedia());
455 }
456
457 private void putScreenInCallMode(final Set<Media> media) {
458 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
459 if (Media.audioOnly(media)) {
460 final JingleRtpConnection rtpConnection =
461 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
462 final AppRTCAudioManager audioManager =
463 rtpConnection == null ? null : rtpConnection.getAudioManager();
464 if (audioManager == null
465 || audioManager.getSelectedAudioDevice()
466 == AppRTCAudioManager.AudioDevice.EARPIECE) {
467 acquireProximityWakeLock();
468 }
469 }
470 lockOrientation(media);
471 }
472
473 private void lockOrientation(final Set<Media> media) {
474 if (Media.audioOnly(media)) {
475 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
476 } else {
477 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
478 }
479 }
480
481 @SuppressLint("WakelockTimeout")
482 private void acquireProximityWakeLock() {
483 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
484 if (powerManager == null) {
485 Log.e(Config.LOGTAG, "power manager not available");
486 return;
487 }
488 if (isFinishing()) {
489 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
490 return;
491 }
492 if (this.mProximityWakeLock == null) {
493 this.mProximityWakeLock =
494 powerManager.newWakeLock(
495 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
496 }
497 if (!this.mProximityWakeLock.isHeld()) {
498 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
499 this.mProximityWakeLock.acquire();
500 }
501 }
502
503 private void releaseProximityWakeLock() {
504 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
505 Log.d(Config.LOGTAG, "releasing proximity wake lock");
506 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
507 this.mProximityWakeLock = null;
508 }
509 }
510
511 private void putProximityWakeLockInProperState(
512 final AppRTCAudioManager.AudioDevice audioDevice) {
513 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
514 acquireProximityWakeLock();
515 } else {
516 releaseProximityWakeLock();
517 }
518 }
519
520 @Override
521 protected void refreshUiReal() {}
522
523 @Override
524 public void onNewIntent(final Intent intent) {
525 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
526 super.onNewIntent(intent);
527 setIntent(intent);
528 if (xmppConnectionService == null) {
529 Log.d(
530 Config.LOGTAG,
531 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
532 return;
533 }
534 final Account account = extractAccount(intent);
535 final String action = intent.getAction();
536 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
537 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
538 if (sessionId != null) {
539 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
540 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
541 return;
542 }
543 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
544 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
545 requestPermissionsAndAcceptCall();
546 resetIntent(intent.getExtras());
547 }
548 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
549 proposeJingleRtpSession(account, with, actionToMedia(action));
550 setWith(account.getRoster().getContact(with), null);
551 } else {
552 throw new IllegalStateException("received onNewIntent without sessionId");
553 }
554 }
555
556 @Override
557 void onBackendConnected() {
558 final Intent intent = getIntent();
559 final String action = intent.getAction();
560 final Account account = extractAccount(intent);
561 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
562 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
563 if (sessionId != null) {
564 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
565 return;
566 }
567 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
568 Log.d(Config.LOGTAG, "intent action was accept");
569 requestPermissionsAndAcceptCall();
570 resetIntent(intent.getExtras());
571 }
572 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
573 proposeJingleRtpSession(account, with, actionToMedia(action));
574 setWith(account.getRoster().getContact(with), null);
575 } else if (Intent.ACTION_VIEW.equals(action)) {
576 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
577 final RtpEndUserState state =
578 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
579 if (state != null) {
580 Log.d(Config.LOGTAG, "restored last state from intent extra");
581 updateButtonConfiguration(state);
582 updateVerifiedShield(false);
583 updateStateDisplay(state);
584 updateIncomingCallScreen(state);
585 invalidateOptionsMenu();
586 }
587 setWith(account.getRoster().getContact(with), state);
588 if (xmppConnectionService
589 .getJingleConnectionManager()
590 .fireJingleRtpConnectionStateUpdates()) {
591 return;
592 }
593 if (END_CARD.contains(state)
594 || xmppConnectionService
595 .getJingleConnectionManager()
596 .hasMatchingProposal(account, with)) {
597 return;
598 }
599 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
600 finish();
601 }
602 }
603
604 private void setWith(final RtpEndUserState state) {
605 setWith(getWith(), state);
606 }
607
608 private void setWith(final Contact contact, final RtpEndUserState state) {
609 binding.with.setText(contact.getDisplayName());
610 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
611 .contains(state)) {
612 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
613 binding.withJid.setVisibility(View.VISIBLE);
614 } else {
615 binding.withJid.setVisibility(View.GONE);
616 }
617 }
618
619 private void proposeJingleRtpSession(
620 final Account account, final Jid with, final Set<Media> media) {
621 checkMicrophoneAvailabilityAsync();
622 if (with.isBareJid()) {
623 xmppConnectionService
624 .getJingleConnectionManager()
625 .proposeJingleRtpSession(account, with, media);
626 } else {
627 final String sessionId =
628 xmppConnectionService
629 .getJingleConnectionManager()
630 .initializeRtpSession(account, with, media);
631 initializeActivityWithRunningRtpSession(account, with, sessionId);
632 resetIntent(account, with, sessionId);
633 }
634 putScreenInCallMode(media);
635 }
636
637 @Override
638 public void onRequestPermissionsResult(
639 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
640 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
641 final PermissionUtils.PermissionResult permissionResult =
642 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
643 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
644 if (requestCode == REQUEST_ACCEPT_CALL) {
645 checkRecorderAndAcceptCall();
646 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
647 acceptContentAdd();
648 } else if (requestCode == REQUEST_ADD_CONTENT) {
649 switchToVideo();
650 }
651 } else {
652 @StringRes int res;
653 final String firstDenied =
654 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
655 if (firstDenied == null) {
656 return;
657 }
658 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
659 res = R.string.no_microphone_permission;
660 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
661 res = R.string.no_camera_permission;
662 } else {
663 throw new IllegalStateException("Invalid permission result request");
664 }
665 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
666 .show();
667 }
668 }
669
670 @Override
671 public void onStart() {
672 super.onStart();
673 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
674 this.binding.remoteVideo.setOnAspectRatioChanged(this);
675 }
676
677 @Override
678 public void onStop() {
679 mHandler.removeCallbacks(mTickExecutor);
680 binding.remoteVideo.release();
681 binding.remoteVideo.setOnAspectRatioChanged(null);
682 binding.localVideo.release();
683 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
684 final JingleRtpConnection jingleRtpConnection =
685 weakReference == null ? null : weakReference.get();
686 if (jingleRtpConnection != null) {
687 releaseVideoTracks(jingleRtpConnection);
688 }
689 releaseProximityWakeLock();
690 super.onStop();
691 }
692
693 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
694 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
695 if (remoteVideo.isPresent()) {
696 remoteVideo.get().removeSink(binding.remoteVideo);
697 }
698 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
699 if (localVideo.isPresent()) {
700 localVideo.get().removeSink(binding.localVideo);
701 }
702 }
703
704 @Override
705 public void onBackPressed() {
706 if (isConnected()) {
707 if (switchToPictureInPicture()) {
708 return;
709 }
710 } else {
711 endCall();
712 }
713 super.onBackPressed();
714 }
715
716 @Override
717 public void onUserLeaveHint() {
718 super.onUserLeaveHint();
719 if (switchToPictureInPicture()) {
720 return;
721 }
722 // TODO apparently this method is not getting called on Android 10 when using the task
723 // switcher
724 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
725 retractSessionProposal();
726 }
727 }
728
729 private boolean isConnected() {
730 final JingleRtpConnection connection =
731 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
732 final RtpEndUserState endUserState =
733 connection == null ? null : connection.getEndUserState();
734 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
735 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
736 }
737
738 private boolean switchToPictureInPicture() {
739 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
740 if (shouldBePictureInPicture()) {
741 startPictureInPicture();
742 return true;
743 }
744 }
745 return false;
746 }
747
748 @RequiresApi(api = Build.VERSION_CODES.O)
749 private void startPictureInPicture() {
750 try {
751 final Rational rational = this.binding.remoteVideo.getAspectRatio();
752 final Rational clippedRational = Rationals.clip(rational);
753 Log.d(
754 Config.LOGTAG,
755 "suggested rational " + rational + ". clipped to " + clippedRational);
756 enterPictureInPictureMode(
757 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
758 } catch (final IllegalStateException e) {
759 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
760 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
761 }
762 }
763
764 @Override
765 public void onAspectRatioChanged(final Rational rational) {
766 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
767 final Rational clippedRational = Rationals.clip(rational);
768 Log.d(
769 Config.LOGTAG,
770 "suggested rational after aspect ratio change "
771 + rational
772 + ". clipped to "
773 + clippedRational);
774 setPictureInPictureParams(
775 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
776 }
777 }
778
779 private boolean deviceSupportsPictureInPicture() {
780 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
781 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
782 } else {
783 return false;
784 }
785 }
786
787 private boolean shouldBePictureInPicture() {
788 try {
789 final JingleRtpConnection rtpConnection = requireRtpConnection();
790 return rtpConnection.getMedia().contains(Media.VIDEO)
791 && Arrays.asList(
792 RtpEndUserState.ACCEPTING_CALL,
793 RtpEndUserState.CONNECTING,
794 RtpEndUserState.CONNECTED)
795 .contains(rtpConnection.getEndUserState());
796 } catch (final IllegalStateException e) {
797 return false;
798 }
799 }
800
801 private boolean initializeActivityWithRunningRtpSession(
802 final Account account, Jid with, String sessionId) {
803 final WeakReference<JingleRtpConnection> reference =
804 xmppConnectionService
805 .getJingleConnectionManager()
806 .findJingleRtpConnection(account, with, sessionId);
807 if (reference == null || reference.get() == null) {
808 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
809 xmppConnectionService
810 .getJingleConnectionManager()
811 .getTerminalSessionState(with, sessionId);
812 if (terminatedRtpSession == null) {
813 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
814 finish();
815 return true;
816 }
817 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
818 return true;
819 }
820 this.rtpConnectionReference = reference;
821 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
822 final boolean verified = requireRtpConnection().isVerified();
823 if (currentState == RtpEndUserState.ENDED) {
824 finish();
825 return true;
826 }
827 final Set<Media> media = getMedia();
828 final ContentAddition contentAddition = getPendingContentAddition();
829 if (currentState == RtpEndUserState.INCOMING_CALL) {
830 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
831 }
832 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
833 requireRtpConnection().getState())) {
834 putScreenInCallMode();
835 }
836 setWith(currentState);
837 updateVideoViews(currentState);
838 updateStateDisplay(currentState, media, contentAddition);
839 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
840 updateButtonConfiguration(currentState, media, contentAddition);
841 updateIncomingCallScreen(currentState);
842 invalidateOptionsMenu();
843 return false;
844 }
845
846 private void initializeWithTerminatedSessionState(
847 final Account account,
848 final Jid with,
849 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
850 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
851 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
852 finish();
853 return;
854 }
855 final RtpEndUserState state = terminatedRtpSession.state;
856 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
857 updateButtonConfiguration(state);
858 updateStateDisplay(state);
859 updateIncomingCallScreen(state);
860 updateCallDuration();
861 updateVerifiedShield(false);
862 invalidateOptionsMenu();
863 setWith(account.getRoster().getContact(with), state);
864 }
865
866 private void reInitializeActivityWithRunningRtpSession(
867 final Account account, Jid with, String sessionId) {
868 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
869 resetIntent(account, with, sessionId);
870 }
871
872 private void resetIntent(final Account account, final Jid with, final String sessionId) {
873 final Intent intent = new Intent(Intent.ACTION_VIEW);
874 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
875 intent.putExtra(EXTRA_WITH, with.toEscapedString());
876 intent.putExtra(EXTRA_SESSION_ID, sessionId);
877 setIntent(intent);
878 }
879
880 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
881 surfaceViewRenderer.setVisibility(View.VISIBLE);
882 try {
883 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
884 } catch (final IllegalStateException e) {
885 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
886 }
887 surfaceViewRenderer.setEnableHardwareScaler(true);
888 }
889
890 private void updateStateDisplay(final RtpEndUserState state) {
891 updateStateDisplay(state, Collections.emptySet(), null);
892 }
893
894 private void updateStateDisplay(
895 final RtpEndUserState state,
896 final Set<Media> media,
897 final ContentAddition contentAddition) {
898 switch (state) {
899 case INCOMING_CALL -> {
900 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
901 if (media.contains(Media.VIDEO)) {
902 setTitle(R.string.rtp_state_incoming_video_call);
903 } else {
904 setTitle(R.string.rtp_state_incoming_call);
905 }
906 }
907 case INCOMING_CONTENT_ADD -> {
908 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
909 setTitle(R.string.rtp_state_content_add_video);
910 } else {
911 setTitle(R.string.rtp_state_content_add);
912 }
913 }
914 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
915 case CONNECTED -> setTitle(R.string.rtp_state_connected);
916 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
917 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
918 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
919 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
920 case RINGING -> setTitle(R.string.rtp_state_ringing);
921 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
922 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
923 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
924 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
925 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
926 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
927 case ENDED -> throw new IllegalStateException(
928 "Activity should have called finishAndReleaseWakeLock();");
929 default -> throw new IllegalStateException(
930 String.format("State %s has not been handled in UI", state));
931 }
932 }
933
934 private void updateVerifiedShield(final boolean verified) {
935 if (isPictureInPicture()) {
936 this.binding.verified.setVisibility(View.GONE);
937 return;
938 }
939 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
940 }
941
942 private void updateIncomingCallScreen(final RtpEndUserState state) {
943 updateIncomingCallScreen(state, null);
944 }
945
946 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
947 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
948 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
949 if (show) {
950 binding.contactPhoto.setVisibility(View.VISIBLE);
951 if (contact == null) {
952 AvatarWorkerTask.loadAvatar(
953 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
954 } else {
955 AvatarWorkerTask.loadAvatar(
956 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
957 }
958 } else {
959 binding.contactPhoto.setVisibility(View.GONE);
960 }
961 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
962 binding.usingAccount.setVisibility(View.VISIBLE);
963 binding.usingAccount.setText(
964 getString(
965 R.string.using_account,
966 account.getJid().asBareJid().toEscapedString()));
967 } else {
968 binding.usingAccount.setVisibility(View.GONE);
969 binding.contactPhoto.setVisibility(View.GONE);
970 }
971 }
972
973 private Set<Media> getMedia() {
974 return requireRtpConnection().getMedia();
975 }
976
977 public ContentAddition getPendingContentAddition() {
978 return requireRtpConnection().getPendingContentAddition();
979 }
980
981 private void updateButtonConfiguration(final RtpEndUserState state) {
982 updateButtonConfiguration(state, Collections.emptySet(), null);
983 }
984
985 @SuppressLint("RestrictedApi")
986 private void updateButtonConfiguration(
987 final RtpEndUserState state,
988 final Set<Media> media,
989 final ContentAddition contentAddition) {
990 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
991 this.binding.rejectCall.setVisibility(View.INVISIBLE);
992 this.binding.endCall.setVisibility(View.INVISIBLE);
993 this.binding.acceptCall.setVisibility(View.INVISIBLE);
994 } else if (state == RtpEndUserState.INCOMING_CALL) {
995 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
996 this.binding.rejectCall.setOnClickListener(this::rejectCall);
997 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
998 this.binding.rejectCall.setVisibility(View.VISIBLE);
999 this.binding.endCall.setVisibility(View.INVISIBLE);
1000 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
1001 this.binding.acceptCall.setOnClickListener(this::acceptCall);
1002 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
1003 this.binding.acceptCall.setVisibility(View.VISIBLE);
1004 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
1005 this.binding.rejectCall.setContentDescription(
1006 getString(R.string.reject_switch_to_video));
1007 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
1008 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1009 this.binding.rejectCall.setVisibility(View.VISIBLE);
1010 this.binding.endCall.setVisibility(View.INVISIBLE);
1011 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1012 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1013 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
1014 this.binding.acceptCall.setVisibility(View.VISIBLE);
1015 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
1016 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1017 this.binding.rejectCall.setOnClickListener(this::exit);
1018 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1019 this.binding.rejectCall.setVisibility(View.VISIBLE);
1020 this.binding.endCall.setVisibility(View.INVISIBLE);
1021 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1022 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1023 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1024 this.binding.acceptCall.setVisibility(View.VISIBLE);
1025 } else if (asList(
1026 RtpEndUserState.CONNECTIVITY_ERROR,
1027 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1028 RtpEndUserState.APPLICATION_ERROR,
1029 RtpEndUserState.RETRACTED,
1030 RtpEndUserState.SECURITY_ERROR)
1031 .contains(state)) {
1032 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1033 this.binding.rejectCall.setOnClickListener(this::exit);
1034 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1035 this.binding.rejectCall.setVisibility(View.VISIBLE);
1036 this.binding.endCall.setVisibility(View.INVISIBLE);
1037 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1038 this.binding.acceptCall.setOnClickListener(this::retry);
1039 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1040 this.binding.acceptCall.setVisibility(View.VISIBLE);
1041 } else {
1042 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1043 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1044 this.binding.endCall.setOnClickListener(this::endCall);
1045 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1046 this.binding.endCall.setVisibility(View.VISIBLE);
1047 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1048 }
1049 updateInCallButtonConfiguration(state, media);
1050 }
1051
1052 private boolean isPictureInPicture() {
1053 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1054 return isInPictureInPictureMode();
1055 } else {
1056 return false;
1057 }
1058 }
1059
1060 private void updateInCallButtonConfiguration() {
1061 updateInCallButtonConfiguration(
1062 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1063 }
1064
1065 @SuppressLint("RestrictedApi")
1066 private void updateInCallButtonConfiguration(
1067 final RtpEndUserState state, final Set<Media> media) {
1068 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1069 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1070 if (media.contains(Media.VIDEO)) {
1071 final JingleRtpConnection rtpConnection = requireRtpConnection();
1072 updateInCallButtonConfigurationVideo(
1073 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1074 } else {
1075 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1076 updateInCallButtonConfigurationSpeaker(
1077 audioManager.getSelectedAudioDevice(),
1078 audioManager.getAudioDevices().size());
1079 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1080 }
1081 if (media.contains(Media.AUDIO)) {
1082 updateInCallButtonConfigurationMicrophone(
1083 requireRtpConnection().isMicrophoneEnabled());
1084 } else {
1085 this.binding.inCallActionLeft.setVisibility(View.GONE);
1086 }
1087 } else {
1088 this.binding.inCallActionLeft.setVisibility(View.GONE);
1089 this.binding.inCallActionRight.setVisibility(View.GONE);
1090 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1091 }
1092 }
1093
1094 @SuppressLint("RestrictedApi")
1095 private void updateInCallButtonConfigurationSpeaker(
1096 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1097 switch (selectedAudioDevice) {
1098 case EARPIECE -> {
1099 this.binding.inCallActionRight.setImageResource(
1100 R.drawable.ic_volume_off_black_24dp);
1101 if (numberOfChoices >= 2) {
1102 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1103 } else {
1104 this.binding.inCallActionRight.setOnClickListener(null);
1105 this.binding.inCallActionRight.setClickable(false);
1106 }
1107 }
1108 case WIRED_HEADSET -> {
1109 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1110 this.binding.inCallActionRight.setOnClickListener(null);
1111 this.binding.inCallActionRight.setClickable(false);
1112 }
1113 case SPEAKER_PHONE -> {
1114 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1115 if (numberOfChoices >= 2) {
1116 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1117 } else {
1118 this.binding.inCallActionRight.setOnClickListener(null);
1119 this.binding.inCallActionRight.setClickable(false);
1120 }
1121 }
1122 case BLUETOOTH -> {
1123 this.binding.inCallActionRight.setImageResource(
1124 R.drawable.ic_bluetooth_audio_black_24dp);
1125 this.binding.inCallActionRight.setOnClickListener(null);
1126 this.binding.inCallActionRight.setClickable(false);
1127 }
1128 }
1129 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1130 }
1131
1132 @SuppressLint("RestrictedApi")
1133 private void updateInCallButtonConfigurationVideo(
1134 final boolean videoEnabled, final boolean isCameraSwitchable) {
1135 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1136 if (isCameraSwitchable) {
1137 this.binding.inCallActionFarRight.setImageResource(
1138 R.drawable.ic_flip_camera_android_black_24dp);
1139 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1140 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1141 } else {
1142 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1143 }
1144 if (videoEnabled) {
1145 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1146 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1147 } else {
1148 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1149 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1150 }
1151 }
1152
1153 private void switchCamera(final View view) {
1154 Futures.addCallback(
1155 requireRtpConnection().switchCamera(),
1156 new FutureCallback<>() {
1157 @Override
1158 public void onSuccess(@Nullable Boolean isFrontCamera) {
1159 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1160 }
1161
1162 @Override
1163 public void onFailure(@NonNull final Throwable throwable) {
1164 Log.d(
1165 Config.LOGTAG,
1166 "could not switch camera",
1167 Throwables.getRootCause(throwable));
1168 Toast.makeText(
1169 RtpSessionActivity.this,
1170 R.string.could_not_switch_camera,
1171 Toast.LENGTH_LONG)
1172 .show();
1173 }
1174 },
1175 MainThreadExecutor.getInstance());
1176 }
1177
1178 private void enableVideo(View view) {
1179 try {
1180 requireRtpConnection().setVideoEnabled(true);
1181 } catch (final IllegalStateException e) {
1182 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1183 return;
1184 }
1185 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1186 }
1187
1188 private void disableVideo(View view) {
1189 final JingleRtpConnection rtpConnection = requireRtpConnection();
1190 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1191 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1192 rtpConnection.retractContentAdd();
1193 return;
1194 }
1195 requireRtpConnection().setVideoEnabled(false);
1196 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1197 }
1198
1199 @SuppressLint("RestrictedApi")
1200 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1201 if (microphoneEnabled) {
1202 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1203 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1204 } else {
1205 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1206 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1207 }
1208 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1209 }
1210
1211 private void updateCallDuration() {
1212 final JingleRtpConnection connection =
1213 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1214 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1215 this.binding.duration.setVisibility(View.GONE);
1216 return;
1217 }
1218 if (connection.zeroDuration()) {
1219 this.binding.duration.setVisibility(View.GONE);
1220 } else {
1221 this.binding.duration.setText(
1222 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1223 this.binding.duration.setVisibility(View.VISIBLE);
1224 }
1225 }
1226
1227 private void updateVideoViews(final RtpEndUserState state) {
1228 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1229 binding.localVideo.setVisibility(View.GONE);
1230 binding.localVideo.release();
1231 binding.remoteVideoWrapper.setVisibility(View.GONE);
1232 binding.remoteVideo.release();
1233 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1234 if (isPictureInPicture()) {
1235 binding.appBarLayout.setVisibility(View.GONE);
1236 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1237 if (Arrays.asList(
1238 RtpEndUserState.APPLICATION_ERROR,
1239 RtpEndUserState.CONNECTIVITY_ERROR,
1240 RtpEndUserState.SECURITY_ERROR)
1241 .contains(state)) {
1242 binding.pipWarning.setVisibility(View.VISIBLE);
1243 binding.pipWaiting.setVisibility(View.GONE);
1244 } else {
1245 binding.pipWarning.setVisibility(View.GONE);
1246 binding.pipWaiting.setVisibility(View.GONE);
1247 }
1248 } else {
1249 binding.appBarLayout.setVisibility(View.VISIBLE);
1250 binding.pipPlaceholder.setVisibility(View.GONE);
1251 }
1252 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1253 return;
1254 }
1255 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1256 binding.localVideo.setVisibility(View.GONE);
1257 binding.remoteVideoWrapper.setVisibility(View.GONE);
1258 binding.appBarLayout.setVisibility(View.GONE);
1259 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1260 binding.pipWarning.setVisibility(View.GONE);
1261 binding.pipWaiting.setVisibility(View.VISIBLE);
1262 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1263 return;
1264 }
1265 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1266 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1267 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1268 // paint local view over remote view
1269 binding.localVideo.setZOrderMediaOverlay(true);
1270 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1271 addSink(localVideoTrack.get(), binding.localVideo);
1272 } else {
1273 binding.localVideo.setVisibility(View.GONE);
1274 }
1275 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1276 if (remoteVideoTrack.isPresent()) {
1277 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1278 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1279 binding.remoteVideo.setScalingType(
1280 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1281 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1282 if (state == RtpEndUserState.CONNECTED) {
1283 binding.appBarLayout.setVisibility(View.GONE);
1284 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1285 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1286 } else {
1287 binding.appBarLayout.setVisibility(View.VISIBLE);
1288 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1289 binding.remoteVideoWrapper.setVisibility(View.GONE);
1290 }
1291 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1292 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1293 } else {
1294 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1295 }
1296 } else {
1297 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1298 binding.remoteVideoWrapper.setVisibility(View.GONE);
1299 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1300 }
1301 }
1302
1303 private Optional<VideoTrack> getLocalVideoTrack() {
1304 final JingleRtpConnection connection =
1305 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1306 if (connection == null) {
1307 return Optional.absent();
1308 }
1309 return connection.getLocalVideoTrack();
1310 }
1311
1312 private Optional<VideoTrack> getRemoteVideoTrack() {
1313 final JingleRtpConnection connection =
1314 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1315 if (connection == null) {
1316 return Optional.absent();
1317 }
1318 return connection.getRemoteVideoTrack();
1319 }
1320
1321 private void disableMicrophone(View view) {
1322 final JingleRtpConnection rtpConnection = requireRtpConnection();
1323 if (rtpConnection.setMicrophoneEnabled(false)) {
1324 updateInCallButtonConfiguration();
1325 }
1326 }
1327
1328 private void enableMicrophone(View view) {
1329 final JingleRtpConnection rtpConnection = requireRtpConnection();
1330 if (rtpConnection.setMicrophoneEnabled(true)) {
1331 updateInCallButtonConfiguration();
1332 }
1333 }
1334
1335 private void switchToEarpiece(View view) {
1336 requireRtpConnection()
1337 .getAudioManager()
1338 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1339 acquireProximityWakeLock();
1340 }
1341
1342 private void switchToSpeaker(View view) {
1343 requireRtpConnection()
1344 .getAudioManager()
1345 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1346 releaseProximityWakeLock();
1347 }
1348
1349 private void retry(View view) {
1350 final Intent intent = getIntent();
1351 final Account account = extractAccount(intent);
1352 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1353 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1354 final String action = intent.getAction();
1355 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1356 this.rtpConnectionReference = null;
1357 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1358 proposeJingleRtpSession(account, with, media);
1359 }
1360
1361 private void exit(final View view) {
1362 finish();
1363 }
1364
1365 private void recordVoiceMail(final View view) {
1366 final Intent intent = getIntent();
1367 final Account account = extractAccount(intent);
1368 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1369 final Conversation conversation =
1370 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1371 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1372 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1373 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1374 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1375 launchIntent.putExtra(
1376 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1377 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1378 startActivity(launchIntent);
1379 finish();
1380 }
1381
1382 private Contact getWith() {
1383 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1384 final Account account = id.account;
1385 return account.getRoster().getContact(id.with);
1386 }
1387
1388 private JingleRtpConnection requireRtpConnection() {
1389 final JingleRtpConnection connection =
1390 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1391 if (connection == null) {
1392 throw new IllegalStateException("No RTP connection found");
1393 }
1394 return connection;
1395 }
1396
1397 @Override
1398 public void onJingleRtpConnectionUpdate(
1399 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1400 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1401 if (END_CARD.contains(state)) {
1402 Log.d(Config.LOGTAG, "end card reached");
1403 releaseProximityWakeLock();
1404 runOnUiThread(
1405 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1406 }
1407 if (with.isBareJid()) {
1408 updateRtpSessionProposalState(account, with, state);
1409 return;
1410 }
1411 if (emptyReference(this.rtpConnectionReference)) {
1412 if (END_CARD.contains(state)) {
1413 Log.d(Config.LOGTAG, "not reinitializing session");
1414 return;
1415 }
1416 // this happens when going from proposed session to actual session
1417 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1418 return;
1419 }
1420 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1421 final boolean verified = requireRtpConnection().isVerified();
1422 final Set<Media> media = getMedia();
1423 lockOrientation(media);
1424 final ContentAddition contentAddition = getPendingContentAddition();
1425 final Contact contact = getWith();
1426 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1427 if (state == RtpEndUserState.ENDED) {
1428 finish();
1429 return;
1430 }
1431 runOnUiThread(
1432 () -> {
1433 updateStateDisplay(state, media, contentAddition);
1434 updateVerifiedShield(
1435 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1436 updateButtonConfiguration(state, media, contentAddition);
1437 updateVideoViews(state);
1438 updateIncomingCallScreen(state, contact);
1439 invalidateOptionsMenu();
1440 });
1441 if (END_CARD.contains(state)) {
1442 final JingleRtpConnection rtpConnection = requireRtpConnection();
1443 resetIntent(account, with, state, rtpConnection.getMedia());
1444 releaseVideoTracks(rtpConnection);
1445 this.rtpConnectionReference = null;
1446 }
1447 } else {
1448 Log.d(Config.LOGTAG, "received update for other rtp session");
1449 }
1450 }
1451
1452 @Override
1453 public void onAudioDeviceChanged(
1454 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1455 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1456 Log.d(
1457 Config.LOGTAG,
1458 "onAudioDeviceChanged in activity: selected:"
1459 + selectedAudioDevice
1460 + ", available:"
1461 + availableAudioDevices);
1462 try {
1463 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1464 final Set<Media> media = getMedia();
1465 if (END_CARD.contains(endUserState)) {
1466 Log.d(
1467 Config.LOGTAG,
1468 "onAudioDeviceChanged() nothing to do because end card has been reached");
1469 } else {
1470 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1471 final AppRTCAudioManager audioManager =
1472 requireRtpConnection().getAudioManager();
1473 updateInCallButtonConfigurationSpeaker(
1474 audioManager.getSelectedAudioDevice(),
1475 audioManager.getAudioDevices().size());
1476 }
1477 Log.d(
1478 Config.LOGTAG,
1479 "put proximity wake lock into proper state after device update");
1480 putProximityWakeLockInProperState(selectedAudioDevice);
1481 }
1482 } catch (final IllegalStateException e) {
1483 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1484 }
1485 }
1486
1487 @Override
1488 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1489 super.onSaveInstanceState(outState);
1490 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1491 }
1492
1493 private void updateRtpSessionProposalState(
1494 final Account account, final Jid with, final RtpEndUserState state) {
1495 final Intent currentIntent = getIntent();
1496 final String withExtra =
1497 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1498 if (withExtra == null) {
1499 return;
1500 }
1501 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1502 runOnUiThread(
1503 () -> {
1504 updateVerifiedShield(false);
1505 updateStateDisplay(state);
1506 updateButtonConfiguration(state);
1507 updateIncomingCallScreen(state);
1508 invalidateOptionsMenu();
1509 });
1510 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1511 }
1512 }
1513
1514 private void resetIntent(final Bundle extras) {
1515 final Intent intent = new Intent(Intent.ACTION_VIEW);
1516 intent.putExtras(extras);
1517 setIntent(intent);
1518 }
1519
1520 private void resetIntent(
1521 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1522 final Intent intent = new Intent(Intent.ACTION_VIEW);
1523 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1524 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1525 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1526 } else {
1527 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1528 }
1529 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1530 intent.putExtra(
1531 EXTRA_LAST_ACTION,
1532 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1533 setIntent(intent);
1534 }
1535
1536 private static boolean emptyReference(final WeakReference<?> weakReference) {
1537 return weakReference == null || weakReference.get() == null;
1538 }
1539}