1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.opengl.GLException;
17import android.os.Build;
18import android.os.Bundle;
19import android.os.Handler;
20import android.os.PowerManager;
21import android.os.SystemClock;
22import android.util.Log;
23import android.util.Rational;
24import android.view.KeyEvent;
25import android.view.Menu;
26import android.view.MenuItem;
27import android.view.View;
28import android.view.WindowManager;
29import android.widget.Toast;
30
31import androidx.annotation.NonNull;
32import androidx.annotation.Nullable;
33import androidx.annotation.RequiresApi;
34import androidx.annotation.StringRes;
35import androidx.databinding.DataBindingUtil;
36
37import com.google.common.base.Optional;
38import com.google.common.base.Preconditions;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.services.CallIntegration;
53import eu.siacs.conversations.services.CallIntegrationConnectionService;
54import eu.siacs.conversations.services.XmppConnectionService;
55import eu.siacs.conversations.ui.util.AvatarWorkerTask;
56import eu.siacs.conversations.ui.util.MainThreadExecutor;
57import eu.siacs.conversations.ui.util.Rationals;
58import eu.siacs.conversations.utils.PermissionUtils;
59import eu.siacs.conversations.utils.TimeFrameUtils;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
62import eu.siacs.conversations.xmpp.jingle.ContentAddition;
63import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
64import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
65import eu.siacs.conversations.xmpp.jingle.Media;
66import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
67import eu.siacs.conversations.xmpp.jingle.RtpCapability;
68import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
69
70import org.webrtc.RendererCommon;
71import org.webrtc.SurfaceViewRenderer;
72import org.webrtc.VideoTrack;
73
74import java.lang.ref.WeakReference;
75import java.util.Arrays;
76import java.util.Collections;
77import java.util.List;
78import java.util.Set;
79
80public class RtpSessionActivity extends XmppActivity
81 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
82 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
83
84 public static final String EXTRA_WITH = "with";
85 public static final String EXTRA_SESSION_ID = "session_id";
86 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
87 public static final String EXTRA_LAST_ACTION = "last_action";
88 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
89 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
90 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
91
92 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
93
94 private static final List<RtpEndUserState> END_CARD =
95 Arrays.asList(
96 RtpEndUserState.APPLICATION_ERROR,
97 RtpEndUserState.SECURITY_ERROR,
98 RtpEndUserState.DECLINED_OR_BUSY,
99 RtpEndUserState.CONTACT_OFFLINE,
100 RtpEndUserState.CONNECTIVITY_ERROR,
101 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
102 RtpEndUserState.RETRACTED);
103 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
104 Arrays.asList(
105 RtpEndUserState.APPLICATION_ERROR,
106 RtpEndUserState.CONNECTIVITY_ERROR,
107 RtpEndUserState.SECURITY_ERROR);
108 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
109 Arrays.asList(
110 RtpEndUserState.CONNECTING,
111 RtpEndUserState.CONNECTED,
112 RtpEndUserState.RECONNECTING,
113 RtpEndUserState.INCOMING_CONTENT_ADD);
114 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
115 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
116 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
117 Arrays.asList(
118 RtpEndUserState.ACCEPTING_CALL,
119 RtpEndUserState.CONNECTING,
120 RtpEndUserState.RECONNECTING);
121 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
122 new ImmutableList.Builder<RtpEndUserState>()
123 .add(RtpEndUserState.FINDING_DEVICE)
124 .add(RtpEndUserState.RINGING)
125 .add(RtpEndUserState.CONNECTING)
126 .addAll(STATES_CONSIDERED_CONNECTED)
127 .build();
128 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
129 private static final int REQUEST_ACCEPT_CALL = 0x1111;
130 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
131 private static final int REQUEST_ADD_CONTENT = 0x1113;
132 private WeakReference<JingleRtpConnection> rtpConnectionReference;
133
134 private ActivityRtpSessionBinding binding;
135 private PowerManager.WakeLock mProximityWakeLock;
136
137 private final Handler mHandler = new Handler();
138 private final Runnable mTickExecutor =
139 new Runnable() {
140 @Override
141 public void run() {
142 updateCallDuration();
143 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
144 }
145 };
146
147 public static Set<Media> actionToMedia(final String action) {
148 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
149 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
150 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
151 return ImmutableSet.of(Media.AUDIO);
152 } else {
153 Log.w(
154 Config.LOGTAG,
155 "actionToMedia can not get media set from unknown action " + action);
156 return Collections.emptySet();
157 }
158 }
159
160 private static void addSink(
161 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
162 try {
163 videoTrack.addSink(surfaceViewRenderer);
164 } catch (final IllegalStateException e) {
165 Log.e(
166 Config.LOGTAG,
167 "possible race condition on trying to display video track. ignoring",
168 e);
169 }
170 }
171
172 @Override
173 public void onCreate(Bundle savedInstanceState) {
174 super.onCreate(savedInstanceState);
175 getWindow()
176 .addFlags(
177 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
178 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
179 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
180 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
181 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
182 setSupportActionBar(binding.toolbar);
183 }
184
185 @Override
186 public boolean onCreateOptionsMenu(final Menu menu) {
187 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
188 final MenuItem help = menu.findItem(R.id.action_help);
189 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
190 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
191 help.setVisible(Config.HELP != null && isHelpButtonVisible());
192 gotoChat.setVisible(isSwitchToConversationVisible());
193 switchToVideo.setVisible(isSwitchToVideoVisible());
194 return super.onCreateOptionsMenu(menu);
195 }
196
197 @Override
198 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
199 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
200 if (xmppConnectionService != null) {
201 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
202 return true;
203 }
204 }
205 }
206 return super.onKeyDown(keyCode, event);
207 }
208
209 private boolean isHelpButtonVisible() {
210 try {
211 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
212 } catch (IllegalStateException e) {
213 final Intent intent = getIntent();
214 final String state =
215 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
216 if (state != null) {
217 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
218 } else {
219 return false;
220 }
221 }
222 }
223
224 private boolean isSwitchToConversationVisible() {
225 final JingleRtpConnection connection =
226 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
227 return connection != null
228 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
229 }
230
231 private boolean isSwitchToVideoVisible() {
232 final JingleRtpConnection connection =
233 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
234 if (connection == null) {
235 return false;
236 }
237 return connection.isSwitchToVideoAvailable();
238 }
239
240 private void switchToConversation() {
241 final Contact contact = getWith();
242 final Conversation conversation =
243 xmppConnectionService.findOrCreateConversation(
244 contact.getAccount(), contact.getJid(), false, true);
245 switchToConversation(conversation);
246 }
247
248 public boolean onOptionsItemSelected(final MenuItem item) {
249 final var itemItem = item.getItemId();
250 if (itemItem == R.id.action_help) {
251 launchHelpInBrowser();
252 return true;
253 } else if (itemItem == R.id.action_goto_chat) {
254 switchToConversation();
255 return true;
256 } else if (itemItem == R.id.action_switch_to_video) {
257 requestPermissionAndSwitchToVideo();
258 return true;
259 } else {
260 return super.onOptionsItemSelected(item);
261 }
262 }
263
264 private void launchHelpInBrowser() {
265 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
266 try {
267 startActivity(intent);
268 } catch (final ActivityNotFoundException e) {
269 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
270 .show();
271 }
272 }
273
274 private void endCall(View view) {
275 endCall();
276 }
277
278 private void endCall() {
279 if (this.rtpConnectionReference == null) {
280 retractSessionProposal();
281 finish();
282 } else {
283 requireRtpConnection().endCall();
284 }
285 }
286
287 private void retractSessionProposal() {
288 final Intent intent = getIntent();
289 final String action = intent.getAction();
290 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
291 final Account account = extractAccount(intent);
292 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
293 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
294 if (!Intent.ACTION_VIEW.equals(action)
295 || state == null
296 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
297 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
298 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
299 }
300 xmppConnectionService
301 .getJingleConnectionManager()
302 .retractSessionProposal(account, with.asBareJid());
303 }
304
305 private void rejectCall(View view) {
306 requireRtpConnection().rejectCall();
307 finish();
308 }
309
310 private void acceptCall(View view) {
311 requestPermissionsAndAcceptCall();
312 }
313
314 private void acceptContentAdd() {
315 try {
316 requireRtpConnection()
317 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
318 } catch (final IllegalStateException e) {
319 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
320 }
321 }
322
323 private void requestPermissionAndSwitchToVideo() {
324 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
325 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
326 switchToVideo();
327 }
328 }
329
330 private void switchToVideo() {
331 try {
332 requireRtpConnection().addMedia(Media.VIDEO);
333 } catch (final IllegalStateException e) {
334 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
335 }
336 }
337
338 private void acceptContentAdd(final ContentAddition contentAddition) {
339 if (contentAddition == null
340 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
341 Log.d(Config.LOGTAG, "ignore press on content-accept button");
342 return;
343 }
344 requestPermissionAndAcceptContentAdd(contentAddition);
345 }
346
347 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
348 final List<String> permissions = permissions(contentAddition.media());
349 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
350 try {
351 requireRtpConnection().acceptContentAdd(contentAddition.summary);
352 } catch (final IllegalStateException e) {
353 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
354 }
355 }
356 }
357
358 private void rejectContentAdd(final View view) {
359 requireRtpConnection().rejectContentAdd();
360 }
361
362 private void requestPermissionsAndAcceptCall() {
363 final List<String> permissions = permissions(getMedia());
364 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
365 putScreenInCallMode();
366 checkRecorderAndAcceptCall();
367 }
368 }
369
370 private List<String> permissions(final Set<Media> media) {
371 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
372 if (media.contains(Media.VIDEO)) {
373 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
374 } else {
375 permissions.add(Manifest.permission.RECORD_AUDIO);
376 }
377 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
378 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
379 }
380 return permissions.build();
381 }
382
383 private void checkRecorderAndAcceptCall() {
384 checkMicrophoneAvailabilityAsync();
385 try {
386 requireRtpConnection().acceptCall();
387 } catch (final IllegalStateException e) {
388 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
389 }
390 }
391
392 private void checkMicrophoneAvailabilityAsync() {
393 new Thread(new MicrophoneAvailabilityCheck(this)).start();
394 }
395
396 private static class MicrophoneAvailabilityCheck implements Runnable {
397
398 private final WeakReference<Activity> activityReference;
399
400 private MicrophoneAvailabilityCheck(final Activity activity) {
401 this.activityReference = new WeakReference<>(activity);
402 }
403
404 @Override
405 public void run() {
406 final long start = SystemClock.elapsedRealtime();
407 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
408 final long stop = SystemClock.elapsedRealtime();
409 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
410 if (isMicrophoneAvailable) {
411 return;
412 }
413 final Activity activity = activityReference.get();
414 if (activity == null) {
415 return;
416 }
417 activity.runOnUiThread(
418 () ->
419 Toast.makeText(
420 activity,
421 R.string.microphone_unavailable,
422 Toast.LENGTH_LONG)
423 .show());
424 }
425 }
426
427 private void putScreenInCallMode() {
428 putScreenInCallMode(requireRtpConnection().getMedia());
429 }
430
431 private void putScreenInCallMode(final Set<Media> media) {
432 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
433 if (Media.audioOnly(media)) {
434 final JingleRtpConnection rtpConnection =
435 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
436 final CallIntegration callIntegration =
437 rtpConnection == null ? null : rtpConnection.getCallIntegration();
438 if (callIntegration == null
439 || callIntegration.getSelectedAudioDevice()
440 == CallIntegration.AudioDevice.EARPIECE) {
441 acquireProximityWakeLock();
442 }
443 }
444 lockOrientation(media);
445 }
446
447 private void lockOrientation(final Set<Media> media) {
448 if (Media.audioOnly(media)) {
449 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
450 } else {
451 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
452 }
453 }
454
455 @SuppressLint("WakelockTimeout")
456 private void acquireProximityWakeLock() {
457 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
458 if (powerManager == null) {
459 Log.e(Config.LOGTAG, "power manager not available");
460 return;
461 }
462 if (isFinishing()) {
463 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
464 return;
465 }
466 if (this.mProximityWakeLock == null) {
467 this.mProximityWakeLock =
468 powerManager.newWakeLock(
469 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
470 }
471 if (!this.mProximityWakeLock.isHeld()) {
472 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
473 this.mProximityWakeLock.acquire();
474 }
475 }
476
477 private void releaseProximityWakeLock() {
478 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
479 Log.d(Config.LOGTAG, "releasing proximity wake lock");
480 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
481 this.mProximityWakeLock = null;
482 }
483 }
484
485 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
486 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
487 acquireProximityWakeLock();
488 } else {
489 releaseProximityWakeLock();
490 }
491 }
492
493 @Override
494 protected void refreshUiReal() {}
495
496 @Override
497 public void onNewIntent(final Intent intent) {
498 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
499 super.onNewIntent(intent);
500 setIntent(intent);
501 if (xmppConnectionService == null) {
502 Log.d(
503 Config.LOGTAG,
504 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
505 return;
506 }
507 final Account account = extractAccount(intent);
508 final String action = intent.getAction();
509 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
510 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
511 if (sessionId != null) {
512 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
513 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
514 return;
515 }
516 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
517 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
518 requestPermissionsAndAcceptCall();
519 resetIntent(intent.getExtras());
520 }
521 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
522 proposeJingleRtpSession(account, with, actionToMedia(action));
523 setWith(account.getRoster().getContact(with), null);
524 } else {
525 throw new IllegalStateException("received onNewIntent without sessionId");
526 }
527 }
528
529 @Override
530 void onBackendConnected() {
531 final Intent intent = getIntent();
532 final String action = intent.getAction();
533 final Account account = extractAccount(intent);
534 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
535 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
536 if (sessionId != null) {
537 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
538 return;
539 }
540 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
541 Log.d(Config.LOGTAG, "intent action was accept");
542 requestPermissionsAndAcceptCall();
543 resetIntent(intent.getExtras());
544 }
545 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
546 proposeJingleRtpSession(account, with, actionToMedia(action));
547 setWith(account.getRoster().getContact(with), null);
548 } else if (Intent.ACTION_VIEW.equals(action)) {
549 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
550 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
551 final Set<Media> media = actionToMedia(lastAction);
552 final RtpEndUserState state =
553 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
554 if (state != null) {
555 Log.d(Config.LOGTAG, "restored last state from intent extra");
556 updateButtonConfiguration(state);
557 updateVerifiedShield(false);
558 updateStateDisplay(state);
559 updateIncomingCallScreen(state);
560 invalidateOptionsMenu();
561 }
562 setWith(account.getRoster().getContact(with), state);
563 if (xmppConnectionService
564 .getJingleConnectionManager()
565 .fireJingleRtpConnectionStateUpdates()) {
566 return;
567 }
568 if (END_CARD.contains(state)) {
569 return;
570 }
571 if (xmppConnectionService
572 .getJingleConnectionManager()
573 .hasMatchingProposal(account, with)) {
574 putScreenInCallMode(media);
575 return;
576 }
577 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
578 finish();
579 }
580 }
581
582 private void setWidth(final RtpEndUserState state) {
583 setWith(getWith(), state);
584 }
585
586 private void setWith(final Contact contact, final RtpEndUserState state) {
587 binding.with.setText(contact.getDisplayName());
588 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
589 .contains(state)) {
590 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
591 binding.withJid.setVisibility(View.VISIBLE);
592 } else {
593 binding.withJid.setVisibility(View.GONE);
594 }
595 }
596
597 private void proposeJingleRtpSession(
598 final Account account, final Jid with, final Set<Media> media) {
599 checkMicrophoneAvailabilityAsync();
600 if (with.isBareJid()) {
601 xmppConnectionService
602 .getJingleConnectionManager()
603 .proposeJingleRtpSession(account, with, media);
604 } else {
605 throw new IllegalStateException(
606 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
607 }
608 putScreenInCallMode(media);
609 }
610
611 @Override
612 public void onRequestPermissionsResult(
613 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
614 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
615 final PermissionUtils.PermissionResult permissionResult =
616 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
617 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
618 if (requestCode == REQUEST_ACCEPT_CALL) {
619 checkRecorderAndAcceptCall();
620 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
621 acceptContentAdd();
622 } else if (requestCode == REQUEST_ADD_CONTENT) {
623 switchToVideo();
624 }
625 } else {
626 @StringRes int res;
627 final String firstDenied =
628 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
629 if (firstDenied == null) {
630 return;
631 }
632 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
633 res = R.string.no_microphone_permission;
634 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
635 res = R.string.no_camera_permission;
636 } else {
637 throw new IllegalStateException("Invalid permission result request");
638 }
639 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
640 .show();
641 }
642 }
643
644 @Override
645 public void onStart() {
646 super.onStart();
647 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
648 this.binding.remoteVideo.setOnAspectRatioChanged(this);
649 }
650
651 @Override
652 public void onStop() {
653 mHandler.removeCallbacks(mTickExecutor);
654 binding.remoteVideo.release();
655 binding.remoteVideo.setOnAspectRatioChanged(null);
656 binding.localVideo.release();
657 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
658 final JingleRtpConnection jingleRtpConnection =
659 weakReference == null ? null : weakReference.get();
660 if (jingleRtpConnection != null) {
661 releaseVideoTracks(jingleRtpConnection);
662 }
663 releaseProximityWakeLock();
664 super.onStop();
665 }
666
667 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
668 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
669 if (remoteVideo.isPresent()) {
670 remoteVideo.get().removeSink(binding.remoteVideo);
671 }
672 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
673 if (localVideo.isPresent()) {
674 localVideo.get().removeSink(binding.localVideo);
675 }
676 }
677
678 @Override
679 public void onBackPressed() {
680 if (isConnected()) {
681 if (switchToPictureInPicture()) {
682 return;
683 }
684 } else {
685 endCall();
686 }
687 super.onBackPressed();
688 }
689
690 @Override
691 public void onUserLeaveHint() {
692 super.onUserLeaveHint();
693 if (switchToPictureInPicture()) {
694 return;
695 }
696 // TODO apparently this method is not getting called on Android 10 when using the task
697 // switcher
698 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
699 retractSessionProposal();
700 }
701 }
702
703 private boolean isConnected() {
704 final JingleRtpConnection connection =
705 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
706 final RtpEndUserState endUserState =
707 connection == null ? null : connection.getEndUserState();
708 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
709 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
710 }
711
712 private boolean switchToPictureInPicture() {
713 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
714 if (shouldBePictureInPicture()) {
715 startPictureInPicture();
716 return true;
717 }
718 }
719 return false;
720 }
721
722 @RequiresApi(api = Build.VERSION_CODES.O)
723 private void startPictureInPicture() {
724 try {
725 final Rational rational = this.binding.remoteVideo.getAspectRatio();
726 final Rational clippedRational = Rationals.clip(rational);
727 Log.d(
728 Config.LOGTAG,
729 "suggested rational " + rational + ". clipped to " + clippedRational);
730 enterPictureInPictureMode(
731 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
732 } catch (final IllegalStateException e) {
733 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
734 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
735 }
736 }
737
738 @Override
739 public void onAspectRatioChanged(final Rational rational) {
740 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
741 final Rational clippedRational = Rationals.clip(rational);
742 Log.d(
743 Config.LOGTAG,
744 "suggested rational after aspect ratio change "
745 + rational
746 + ". clipped to "
747 + clippedRational);
748 setPictureInPictureParams(
749 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
750 }
751 }
752
753 private boolean deviceSupportsPictureInPicture() {
754 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
755 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
756 } else {
757 return false;
758 }
759 }
760
761 private boolean shouldBePictureInPicture() {
762 try {
763 final JingleRtpConnection rtpConnection = requireRtpConnection();
764 return rtpConnection.getMedia().contains(Media.VIDEO)
765 && Arrays.asList(
766 RtpEndUserState.ACCEPTING_CALL,
767 RtpEndUserState.CONNECTING,
768 RtpEndUserState.CONNECTED)
769 .contains(rtpConnection.getEndUserState());
770 } catch (final IllegalStateException e) {
771 return false;
772 }
773 }
774
775 private boolean initializeActivityWithRunningRtpSession(
776 final Account account, Jid with, String sessionId) {
777 final WeakReference<JingleRtpConnection> reference =
778 xmppConnectionService
779 .getJingleConnectionManager()
780 .findJingleRtpConnection(account, with, sessionId);
781 if (reference == null || reference.get() == null) {
782 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
783 xmppConnectionService
784 .getJingleConnectionManager()
785 .getTerminalSessionState(with, sessionId);
786 if (terminatedRtpSession == null) {
787 throw new IllegalStateException(
788 "failed to initialize activity with running rtp session. session not found");
789 }
790 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
791 return true;
792 }
793 this.rtpConnectionReference = reference;
794 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
795 final boolean verified = requireRtpConnection().isVerified();
796 if (currentState == RtpEndUserState.ENDED) {
797 finish();
798 return true;
799 }
800 final Set<Media> media = getMedia();
801 final ContentAddition contentAddition = getPendingContentAddition();
802 if (currentState == RtpEndUserState.INCOMING_CALL) {
803 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
804 }
805 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
806 requireRtpConnection().getState())) {
807 putScreenInCallMode();
808 }
809 setWidth(currentState);
810 updateVideoViews(currentState);
811 updateStateDisplay(currentState, media, contentAddition);
812 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
813 updateButtonConfiguration(currentState, media, contentAddition);
814 updateIncomingCallScreen(currentState);
815 invalidateOptionsMenu();
816 return false;
817 }
818
819 private void initializeWithTerminatedSessionState(
820 final Account account,
821 final Jid with,
822 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
823 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
824 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
825 finish();
826 return;
827 }
828 final RtpEndUserState state = terminatedRtpSession.state;
829 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
830 updateButtonConfiguration(state);
831 updateStateDisplay(state);
832 updateIncomingCallScreen(state);
833 updateCallDuration();
834 updateVerifiedShield(false);
835 invalidateOptionsMenu();
836 setWith(account.getRoster().getContact(with), state);
837 }
838
839 private void reInitializeActivityWithRunningRtpSession(
840 final Account account, Jid with, String sessionId) {
841 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
842 resetIntent(account, with, sessionId);
843 }
844
845 private void resetIntent(final Account account, final Jid with, final String sessionId) {
846 final Intent intent = new Intent(Intent.ACTION_VIEW);
847 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
848 intent.putExtra(EXTRA_WITH, with.toEscapedString());
849 intent.putExtra(EXTRA_SESSION_ID, sessionId);
850 setIntent(intent);
851 }
852
853 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
854 surfaceViewRenderer.setVisibility(View.VISIBLE);
855 try {
856 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
857 } catch (final IllegalStateException ignored) {
858 // SurfaceViewRenderer was already initialized
859 } catch (final RuntimeException e) {
860 if (Throwables.getRootCause(e) instanceof GLException glException) {
861 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
862 }
863 }
864 surfaceViewRenderer.setEnableHardwareScaler(true);
865 }
866
867 private void updateStateDisplay(final RtpEndUserState state) {
868 updateStateDisplay(state, Collections.emptySet(), null);
869 }
870
871 private void updateStateDisplay(
872 final RtpEndUserState state,
873 final Set<Media> media,
874 final ContentAddition contentAddition) {
875 switch (state) {
876 case INCOMING_CALL -> {
877 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
878 if (media.contains(Media.VIDEO)) {
879 setTitle(R.string.rtp_state_incoming_video_call);
880 } else {
881 setTitle(R.string.rtp_state_incoming_call);
882 }
883 }
884 case INCOMING_CONTENT_ADD -> {
885 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
886 setTitle(R.string.rtp_state_content_add_video);
887 } else {
888 setTitle(R.string.rtp_state_content_add);
889 }
890 }
891 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
892 case CONNECTED -> setTitle(R.string.rtp_state_connected);
893 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
894 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
895 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
896 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
897 case RINGING -> setTitle(R.string.rtp_state_ringing);
898 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
899 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
900 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
901 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
902 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
903 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
904 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
905 case ENDED -> throw new IllegalStateException(
906 "Activity should have called finishAndReleaseWakeLock();");
907 default -> throw new IllegalStateException(
908 String.format("State %s has not been handled in UI", state));
909 }
910 }
911
912 private void updateVerifiedShield(final boolean verified) {
913 if (isPictureInPicture()) {
914 this.binding.verified.setVisibility(View.GONE);
915 return;
916 }
917 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
918 }
919
920 private void updateIncomingCallScreen(final RtpEndUserState state) {
921 updateIncomingCallScreen(state, null);
922 }
923
924 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
925 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
926 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
927 if (show) {
928 binding.contactPhoto.setVisibility(View.VISIBLE);
929 if (contact == null) {
930 AvatarWorkerTask.loadAvatar(
931 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
932 } else {
933 AvatarWorkerTask.loadAvatar(
934 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
935 }
936 } else {
937 binding.contactPhoto.setVisibility(View.GONE);
938 }
939 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
940 binding.usingAccount.setVisibility(View.VISIBLE);
941 binding.usingAccount.setText(
942 getString(
943 R.string.using_account,
944 account.getJid().asBareJid().toEscapedString()));
945 } else {
946 binding.usingAccount.setVisibility(View.GONE);
947 binding.contactPhoto.setVisibility(View.GONE);
948 }
949 }
950
951 private Set<Media> getMedia() {
952 return requireRtpConnection().getMedia();
953 }
954
955 public ContentAddition getPendingContentAddition() {
956 return requireRtpConnection().getPendingContentAddition();
957 }
958
959 private void updateButtonConfiguration(final RtpEndUserState state) {
960 updateButtonConfiguration(state, Collections.emptySet(), null);
961 }
962
963 @SuppressLint("RestrictedApi")
964 private void updateButtonConfiguration(
965 final RtpEndUserState state,
966 final Set<Media> media,
967 final ContentAddition contentAddition) {
968 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
969 this.binding.rejectCall.setVisibility(View.INVISIBLE);
970 this.binding.endCall.setVisibility(View.INVISIBLE);
971 this.binding.acceptCall.setVisibility(View.INVISIBLE);
972 } else if (state == RtpEndUserState.INCOMING_CALL) {
973 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
974 this.binding.rejectCall.setOnClickListener(this::rejectCall);
975 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
976 this.binding.rejectCall.setVisibility(View.VISIBLE);
977 this.binding.endCall.setVisibility(View.INVISIBLE);
978 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
979 this.binding.acceptCall.setOnClickListener(this::acceptCall);
980 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
981 this.binding.acceptCall.setVisibility(View.VISIBLE);
982 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
983 this.binding.rejectCall.setContentDescription(
984 getString(R.string.reject_switch_to_video));
985 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
986 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
987 this.binding.rejectCall.setVisibility(View.VISIBLE);
988 this.binding.endCall.setVisibility(View.INVISIBLE);
989 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
990 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
991 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
992 this.binding.acceptCall.setVisibility(View.VISIBLE);
993 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
994 .contains(state)) {
995 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
996 this.binding.rejectCall.setOnClickListener(this::exit);
997 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
998 this.binding.rejectCall.setVisibility(View.VISIBLE);
999 this.binding.endCall.setVisibility(View.INVISIBLE);
1000 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1001 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1002 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1003 this.binding.acceptCall.setVisibility(View.VISIBLE);
1004 } else if (asList(
1005 RtpEndUserState.CONNECTIVITY_ERROR,
1006 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1007 RtpEndUserState.APPLICATION_ERROR,
1008 RtpEndUserState.RETRACTED,
1009 RtpEndUserState.SECURITY_ERROR)
1010 .contains(state)) {
1011 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1012 this.binding.rejectCall.setOnClickListener(this::exit);
1013 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1014 this.binding.rejectCall.setVisibility(View.VISIBLE);
1015 this.binding.endCall.setVisibility(View.INVISIBLE);
1016 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1017 this.binding.acceptCall.setOnClickListener(this::retry);
1018 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1019 this.binding.acceptCall.setVisibility(View.VISIBLE);
1020 } else {
1021 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1022 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1023 this.binding.endCall.setOnClickListener(this::endCall);
1024 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1025 this.binding.endCall.setVisibility(View.VISIBLE);
1026 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1027 }
1028 updateInCallButtonConfiguration(state, media);
1029 }
1030
1031 private boolean isPictureInPicture() {
1032 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1033 return isInPictureInPictureMode();
1034 } else {
1035 return false;
1036 }
1037 }
1038
1039 private void updateInCallButtonConfiguration() {
1040 updateInCallButtonConfiguration(
1041 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1042 }
1043
1044 @SuppressLint("RestrictedApi")
1045 private void updateInCallButtonConfiguration(
1046 final RtpEndUserState state, final Set<Media> media) {
1047 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1048 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1049 if (media.contains(Media.VIDEO)) {
1050 final JingleRtpConnection rtpConnection = requireRtpConnection();
1051 updateInCallButtonConfigurationVideo(
1052 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1053 } else {
1054 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1055 updateInCallButtonConfigurationSpeaker(
1056 callIntegration.getSelectedAudioDevice(),
1057 callIntegration.getAudioDevices().size());
1058 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1059 }
1060 if (media.contains(Media.AUDIO)) {
1061 updateInCallButtonConfigurationMicrophone(
1062 requireRtpConnection().isMicrophoneEnabled());
1063 } else {
1064 this.binding.inCallActionLeft.setVisibility(View.GONE);
1065 }
1066 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1067 && !isPictureInPicture()
1068 && Media.audioOnly(media)) {
1069 final CallIntegration callIntegration = requireCallIntegration();
1070 updateInCallButtonConfigurationSpeaker(
1071 callIntegration.getSelectedAudioDevice(),
1072 callIntegration.getAudioDevices().size());
1073 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1074 } else {
1075 this.binding.inCallActionLeft.setVisibility(View.GONE);
1076 this.binding.inCallActionRight.setVisibility(View.GONE);
1077 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1078 }
1079 }
1080
1081 @SuppressLint("RestrictedApi")
1082 private void updateInCallButtonConfigurationSpeaker(
1083 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1084 switch (selectedAudioDevice) {
1085 case EARPIECE -> {
1086 this.binding.inCallActionRight.setImageResource(
1087 R.drawable.ic_volume_off_black_24dp);
1088 if (numberOfChoices >= 2) {
1089 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1090 } else {
1091 this.binding.inCallActionRight.setOnClickListener(null);
1092 this.binding.inCallActionRight.setClickable(false);
1093 }
1094 }
1095 case WIRED_HEADSET -> {
1096 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1097 this.binding.inCallActionRight.setOnClickListener(null);
1098 this.binding.inCallActionRight.setClickable(false);
1099 }
1100 case SPEAKER_PHONE -> {
1101 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1102 if (numberOfChoices >= 2) {
1103 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1104 } else {
1105 this.binding.inCallActionRight.setOnClickListener(null);
1106 this.binding.inCallActionRight.setClickable(false);
1107 }
1108 }
1109 case BLUETOOTH -> {
1110 this.binding.inCallActionRight.setImageResource(
1111 R.drawable.ic_bluetooth_audio_black_24dp);
1112 this.binding.inCallActionRight.setOnClickListener(null);
1113 this.binding.inCallActionRight.setClickable(false);
1114 }
1115 }
1116 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1117 }
1118
1119 @SuppressLint("RestrictedApi")
1120 private void updateInCallButtonConfigurationVideo(
1121 final boolean videoEnabled, final boolean isCameraSwitchable) {
1122 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1123 if (isCameraSwitchable) {
1124 this.binding.inCallActionFarRight.setImageResource(
1125 R.drawable.ic_flip_camera_android_black_24dp);
1126 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1127 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1128 } else {
1129 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1130 }
1131 if (videoEnabled) {
1132 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1133 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1134 } else {
1135 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1136 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1137 }
1138 }
1139
1140 private void switchCamera(final View view) {
1141 Futures.addCallback(
1142 requireRtpConnection().switchCamera(),
1143 new FutureCallback<>() {
1144 @Override
1145 public void onSuccess(@Nullable Boolean isFrontCamera) {
1146 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1147 }
1148
1149 @Override
1150 public void onFailure(@NonNull final Throwable throwable) {
1151 Log.d(
1152 Config.LOGTAG,
1153 "could not switch camera",
1154 Throwables.getRootCause(throwable));
1155 Toast.makeText(
1156 RtpSessionActivity.this,
1157 R.string.could_not_switch_camera,
1158 Toast.LENGTH_LONG)
1159 .show();
1160 }
1161 },
1162 MainThreadExecutor.getInstance());
1163 }
1164
1165 private void enableVideo(View view) {
1166 try {
1167 requireRtpConnection().setVideoEnabled(true);
1168 } catch (final IllegalStateException e) {
1169 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1170 return;
1171 }
1172 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1173 }
1174
1175 private void disableVideo(View view) {
1176 final JingleRtpConnection rtpConnection = requireRtpConnection();
1177 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1178 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1179 rtpConnection.retractContentAdd();
1180 return;
1181 }
1182 requireRtpConnection().setVideoEnabled(false);
1183 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1184 }
1185
1186 @SuppressLint("RestrictedApi")
1187 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1188 if (microphoneEnabled) {
1189 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1190 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1191 } else {
1192 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1193 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1194 }
1195 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1196 }
1197
1198 private void updateCallDuration() {
1199 final JingleRtpConnection connection =
1200 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1201 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1202 this.binding.duration.setVisibility(View.GONE);
1203 return;
1204 }
1205 if (connection.zeroDuration()) {
1206 this.binding.duration.setVisibility(View.GONE);
1207 } else {
1208 this.binding.duration.setText(
1209 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1210 this.binding.duration.setVisibility(View.VISIBLE);
1211 }
1212 }
1213
1214 private void updateVideoViews(final RtpEndUserState state) {
1215 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1216 binding.localVideo.setVisibility(View.GONE);
1217 binding.localVideo.release();
1218 binding.remoteVideoWrapper.setVisibility(View.GONE);
1219 binding.remoteVideo.release();
1220 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1221 if (isPictureInPicture()) {
1222 binding.appBarLayout.setVisibility(View.GONE);
1223 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1224 if (Arrays.asList(
1225 RtpEndUserState.APPLICATION_ERROR,
1226 RtpEndUserState.CONNECTIVITY_ERROR,
1227 RtpEndUserState.SECURITY_ERROR)
1228 .contains(state)) {
1229 binding.pipWarning.setVisibility(View.VISIBLE);
1230 binding.pipWaiting.setVisibility(View.GONE);
1231 } else {
1232 binding.pipWarning.setVisibility(View.GONE);
1233 binding.pipWaiting.setVisibility(View.GONE);
1234 }
1235 } else {
1236 binding.appBarLayout.setVisibility(View.VISIBLE);
1237 binding.pipPlaceholder.setVisibility(View.GONE);
1238 }
1239 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1240 return;
1241 }
1242 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1243 binding.localVideo.setVisibility(View.GONE);
1244 binding.remoteVideoWrapper.setVisibility(View.GONE);
1245 binding.appBarLayout.setVisibility(View.GONE);
1246 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1247 binding.pipWarning.setVisibility(View.GONE);
1248 binding.pipWaiting.setVisibility(View.VISIBLE);
1249 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1250 return;
1251 }
1252 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1253 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1254 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1255 // paint local view over remote view
1256 binding.localVideo.setZOrderMediaOverlay(true);
1257 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1258 addSink(localVideoTrack.get(), binding.localVideo);
1259 } else {
1260 binding.localVideo.setVisibility(View.GONE);
1261 }
1262 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1263 if (remoteVideoTrack.isPresent()) {
1264 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1265 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1266 binding.remoteVideo.setScalingType(
1267 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1268 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1269 if (state == RtpEndUserState.CONNECTED) {
1270 binding.appBarLayout.setVisibility(View.GONE);
1271 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1272 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1273 } else {
1274 binding.appBarLayout.setVisibility(View.VISIBLE);
1275 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1276 binding.remoteVideoWrapper.setVisibility(View.GONE);
1277 }
1278 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1279 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1280 } else {
1281 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1282 }
1283 } else {
1284 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1285 binding.remoteVideoWrapper.setVisibility(View.GONE);
1286 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1287 }
1288 }
1289
1290 private Optional<VideoTrack> getLocalVideoTrack() {
1291 final JingleRtpConnection connection =
1292 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1293 if (connection == null) {
1294 return Optional.absent();
1295 }
1296 return connection.getLocalVideoTrack();
1297 }
1298
1299 private Optional<VideoTrack> getRemoteVideoTrack() {
1300 final JingleRtpConnection connection =
1301 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1302 if (connection == null) {
1303 return Optional.absent();
1304 }
1305 return connection.getRemoteVideoTrack();
1306 }
1307
1308 private void disableMicrophone(View view) {
1309 final JingleRtpConnection rtpConnection = requireRtpConnection();
1310 if (rtpConnection.setMicrophoneEnabled(false)) {
1311 updateInCallButtonConfiguration();
1312 }
1313 }
1314
1315 private void enableMicrophone(View view) {
1316 final JingleRtpConnection rtpConnection = requireRtpConnection();
1317 if (rtpConnection.setMicrophoneEnabled(true)) {
1318 updateInCallButtonConfiguration();
1319 }
1320 }
1321
1322 private void switchToEarpiece(final View view) {
1323 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1324 acquireProximityWakeLock();
1325 }
1326
1327 private void switchToSpeaker(final View view) {
1328 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1329 releaseProximityWakeLock();
1330 }
1331
1332 private void retry(final View view) {
1333 final Intent intent = getIntent();
1334 final Account account = extractAccount(intent);
1335 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1336 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1337 final String action = intent.getAction();
1338 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1339 this.rtpConnectionReference = null;
1340 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1341 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1342 }
1343
1344 private void exit(final View view) {
1345 finish();
1346 }
1347
1348 private void recordVoiceMail(final View view) {
1349 final Intent intent = getIntent();
1350 final Account account = extractAccount(intent);
1351 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1352 final Conversation conversation =
1353 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1354 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1355 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1356 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1357 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1358 launchIntent.putExtra(
1359 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1360 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1361 startActivity(launchIntent);
1362 finish();
1363 }
1364
1365 private Contact getWith() {
1366 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1367 final Account account = id.account;
1368 return account.getRoster().getContact(id.with);
1369 }
1370
1371 private JingleRtpConnection requireRtpConnection() {
1372 final JingleRtpConnection connection =
1373 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1374 if (connection == null) {
1375 throw new IllegalStateException("No RTP connection found");
1376 }
1377 return connection;
1378 }
1379
1380 private CallIntegration requireCallIntegration() {
1381 return requireOngoingRtpSession().getCallIntegration();
1382 }
1383
1384 private OngoingRtpSession requireOngoingRtpSession() {
1385 final JingleRtpConnection connection =
1386 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1387 if (connection != null) {
1388 return connection;
1389 }
1390 final Intent currentIntent = getIntent();
1391 final String withExtra =
1392 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1393 final var account = extractAccount(currentIntent);
1394 if (withExtra == null) {
1395 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1396 }
1397 final var matching =
1398 xmppConnectionService
1399 .getJingleConnectionManager()
1400 .matchingProposal(account, Jid.of(withExtra));
1401 if (matching.isPresent()) {
1402 return matching.get();
1403 }
1404 throw new IllegalStateException("No matching session proposal");
1405 }
1406
1407 @Override
1408 public void onJingleRtpConnectionUpdate(
1409 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1410 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1411 if (END_CARD.contains(state)) {
1412 Log.d(Config.LOGTAG, "end card reached");
1413 releaseProximityWakeLock();
1414 runOnUiThread(
1415 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1416 }
1417 if (with.isBareJid()) {
1418 updateRtpSessionProposalState(account, with, state);
1419 return;
1420 }
1421 if (emptyReference(this.rtpConnectionReference)) {
1422 if (END_CARD.contains(state)) {
1423 Log.d(Config.LOGTAG, "not reinitializing session");
1424 return;
1425 }
1426 // this happens when going from proposed session to actual session
1427 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1428 return;
1429 }
1430 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1431 final boolean verified = requireRtpConnection().isVerified();
1432 final Set<Media> media = getMedia();
1433 lockOrientation(media);
1434 final ContentAddition contentAddition = getPendingContentAddition();
1435 final Contact contact = getWith();
1436 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1437 if (state == RtpEndUserState.ENDED) {
1438 finish();
1439 return;
1440 }
1441 runOnUiThread(
1442 () -> {
1443 updateStateDisplay(state, media, contentAddition);
1444 updateVerifiedShield(
1445 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1446 updateButtonConfiguration(state, media, contentAddition);
1447 updateVideoViews(state);
1448 updateIncomingCallScreen(state, contact);
1449 invalidateOptionsMenu();
1450 });
1451 if (END_CARD.contains(state)) {
1452 final JingleRtpConnection rtpConnection = requireRtpConnection();
1453 resetIntent(account, with, state, rtpConnection.getMedia());
1454 releaseVideoTracks(rtpConnection);
1455 this.rtpConnectionReference = null;
1456 }
1457 } else {
1458 Log.d(Config.LOGTAG, "received update for other rtp session");
1459 }
1460 }
1461
1462 @Override
1463 public void onAudioDeviceChanged(
1464 final CallIntegration.AudioDevice selectedAudioDevice,
1465 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1466 Log.d(
1467 Config.LOGTAG,
1468 "onAudioDeviceChanged in activity: selected:"
1469 + selectedAudioDevice
1470 + ", available:"
1471 + availableAudioDevices);
1472 try {
1473 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1474 final RtpEndUserState endUserState;
1475 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1476 endUserState = jingleRtpConnection.getEndUserState();
1477 } else {
1478 // for session proposals all end user states are functionally the same
1479 endUserState = RtpEndUserState.RINGING;
1480 }
1481 final Set<Media> media = ongoingRtpSession.getMedia();
1482 if (END_CARD.contains(endUserState)) {
1483 Log.d(
1484 Config.LOGTAG,
1485 "onAudioDeviceChanged() nothing to do because end card has been reached");
1486 } else {
1487 if (Media.audioOnly(media)
1488 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1489 final CallIntegration callIntegration = requireCallIntegration();
1490 updateInCallButtonConfigurationSpeaker(
1491 callIntegration.getSelectedAudioDevice(),
1492 callIntegration.getAudioDevices().size());
1493 }
1494 Log.d(
1495 Config.LOGTAG,
1496 "put proximity wake lock into proper state after device update");
1497 putProximityWakeLockInProperState(selectedAudioDevice);
1498 }
1499 } catch (final IllegalStateException e) {
1500 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1501 }
1502 }
1503
1504 private void updateRtpSessionProposalState(
1505 final Account account, final Jid with, final RtpEndUserState state) {
1506 final Intent currentIntent = getIntent();
1507 final String withExtra =
1508 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1509 if (withExtra == null) {
1510 return;
1511 }
1512 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1513 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1514 runOnUiThread(
1515 () -> {
1516 updateVerifiedShield(false);
1517 updateStateDisplay(state);
1518 updateButtonConfiguration(state, media, null);
1519 updateIncomingCallScreen(state);
1520 invalidateOptionsMenu();
1521 });
1522 resetIntent(account, with, state, media);
1523 }
1524 }
1525
1526 private void resetIntent(final Bundle extras) {
1527 final Intent intent = new Intent(Intent.ACTION_VIEW);
1528 intent.putExtras(extras);
1529 setIntent(intent);
1530 }
1531
1532 private void resetIntent(
1533 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1534 final Intent intent = new Intent(Intent.ACTION_VIEW);
1535 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1536 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1537 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1538 } else {
1539 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1540 }
1541 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1542 intent.putExtra(
1543 EXTRA_LAST_ACTION,
1544 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1545 setIntent(intent);
1546 }
1547
1548 private static boolean emptyReference(final WeakReference<?> weakReference) {
1549 return weakReference == null || weakReference.get() == null;
1550 }
1551}