1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.opengl.GLException;
17import android.os.Build;
18import android.os.Bundle;
19import android.os.Handler;
20import android.os.PowerManager;
21import android.os.SystemClock;
22import android.util.Log;
23import android.util.Rational;
24import android.view.KeyEvent;
25import android.view.Menu;
26import android.view.MenuItem;
27import android.view.View;
28import android.view.WindowManager;
29import android.widget.Toast;
30
31import androidx.annotation.NonNull;
32import androidx.annotation.Nullable;
33import androidx.annotation.RequiresApi;
34import androidx.annotation.StringRes;
35import androidx.databinding.DataBindingUtil;
36
37import com.google.common.base.Optional;
38import com.google.common.base.Preconditions;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.services.CallIntegration;
53import eu.siacs.conversations.services.CallIntegrationConnectionService;
54import eu.siacs.conversations.services.XmppConnectionService;
55import eu.siacs.conversations.ui.util.AvatarWorkerTask;
56import eu.siacs.conversations.ui.util.MainThreadExecutor;
57import eu.siacs.conversations.ui.util.Rationals;
58import eu.siacs.conversations.utils.PermissionUtils;
59import eu.siacs.conversations.utils.TimeFrameUtils;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
62import eu.siacs.conversations.xmpp.jingle.ContentAddition;
63import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
64import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
65import eu.siacs.conversations.xmpp.jingle.Media;
66import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
67import eu.siacs.conversations.xmpp.jingle.RtpCapability;
68import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
69
70import org.webrtc.RendererCommon;
71import org.webrtc.SurfaceViewRenderer;
72import org.webrtc.VideoTrack;
73
74import java.lang.ref.WeakReference;
75import java.util.Arrays;
76import java.util.Collections;
77import java.util.List;
78import java.util.Set;
79
80public class RtpSessionActivity extends XmppActivity
81 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
82 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
83
84 public static final String EXTRA_WITH = "with";
85 public static final String EXTRA_SESSION_ID = "session_id";
86 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
87 public static final String EXTRA_LAST_ACTION = "last_action";
88 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
89 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
90 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
91
92 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
93
94 private static final List<RtpEndUserState> END_CARD =
95 Arrays.asList(
96 RtpEndUserState.APPLICATION_ERROR,
97 RtpEndUserState.SECURITY_ERROR,
98 RtpEndUserState.DECLINED_OR_BUSY,
99 RtpEndUserState.CONTACT_OFFLINE,
100 RtpEndUserState.CONNECTIVITY_ERROR,
101 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
102 RtpEndUserState.RETRACTED);
103 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
104 Arrays.asList(
105 RtpEndUserState.APPLICATION_ERROR,
106 RtpEndUserState.CONNECTIVITY_ERROR,
107 RtpEndUserState.SECURITY_ERROR);
108 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
109 Arrays.asList(
110 RtpEndUserState.CONNECTING,
111 RtpEndUserState.CONNECTED,
112 RtpEndUserState.RECONNECTING,
113 RtpEndUserState.INCOMING_CONTENT_ADD);
114 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
115 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
116 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
117 Arrays.asList(
118 RtpEndUserState.ACCEPTING_CALL,
119 RtpEndUserState.CONNECTING,
120 RtpEndUserState.RECONNECTING);
121 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
122 new ImmutableList.Builder<RtpEndUserState>()
123 .add(RtpEndUserState.FINDING_DEVICE)
124 .add(RtpEndUserState.RINGING)
125 .add(RtpEndUserState.ACCEPTING_CALL)
126 .add(RtpEndUserState.CONNECTING)
127 .addAll(STATES_CONSIDERED_CONNECTED)
128 .build();
129 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
130 private static final int REQUEST_ACCEPT_CALL = 0x1111;
131 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
132 private static final int REQUEST_ADD_CONTENT = 0x1113;
133 private WeakReference<JingleRtpConnection> rtpConnectionReference;
134
135 private ActivityRtpSessionBinding binding;
136 private PowerManager.WakeLock mProximityWakeLock;
137
138 private final Handler mHandler = new Handler();
139 private final Runnable mTickExecutor =
140 new Runnable() {
141 @Override
142 public void run() {
143 updateCallDuration();
144 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
145 }
146 };
147
148 public static Set<Media> actionToMedia(final String action) {
149 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
150 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
151 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
152 return ImmutableSet.of(Media.AUDIO);
153 } else {
154 Log.w(
155 Config.LOGTAG,
156 "actionToMedia can not get media set from unknown action " + action);
157 return Collections.emptySet();
158 }
159 }
160
161 private static void addSink(
162 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
163 try {
164 videoTrack.addSink(surfaceViewRenderer);
165 } catch (final IllegalStateException e) {
166 Log.e(
167 Config.LOGTAG,
168 "possible race condition on trying to display video track. ignoring",
169 e);
170 }
171 }
172
173 @Override
174 public void onCreate(Bundle savedInstanceState) {
175 super.onCreate(savedInstanceState);
176 getWindow()
177 .addFlags(
178 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
179 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
180 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
181 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
182 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
183 setSupportActionBar(binding.toolbar);
184 }
185
186 @Override
187 public boolean onCreateOptionsMenu(final Menu menu) {
188 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
189 final MenuItem help = menu.findItem(R.id.action_help);
190 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
191 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
192 help.setVisible(Config.HELP != null && isHelpButtonVisible());
193 gotoChat.setVisible(isSwitchToConversationVisible());
194 switchToVideo.setVisible(isSwitchToVideoVisible());
195 return super.onCreateOptionsMenu(menu);
196 }
197
198 @Override
199 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
200 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
201 if (xmppConnectionService != null) {
202 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
203 return true;
204 }
205 }
206 }
207 return super.onKeyDown(keyCode, event);
208 }
209
210 private boolean isHelpButtonVisible() {
211 try {
212 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
213 } catch (IllegalStateException e) {
214 final Intent intent = getIntent();
215 final String state =
216 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
217 if (state != null) {
218 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
219 } else {
220 return false;
221 }
222 }
223 }
224
225 private boolean isSwitchToConversationVisible() {
226 final JingleRtpConnection connection =
227 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
228 return connection != null
229 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
230 }
231
232 private boolean isSwitchToVideoVisible() {
233 final JingleRtpConnection connection =
234 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
235 if (connection == null) {
236 return false;
237 }
238 return connection.isSwitchToVideoAvailable();
239 }
240
241 private void switchToConversation() {
242 final Contact contact = getWith();
243 final Conversation conversation =
244 xmppConnectionService.findOrCreateConversation(
245 contact.getAccount(), contact.getJid(), false, true);
246 switchToConversation(conversation);
247 }
248
249 public boolean onOptionsItemSelected(final MenuItem item) {
250 final var itemItem = item.getItemId();
251 if (itemItem == R.id.action_help) {
252 launchHelpInBrowser();
253 return true;
254 } else if (itemItem == R.id.action_goto_chat) {
255 switchToConversation();
256 return true;
257 } else if (itemItem == R.id.action_switch_to_video) {
258 requestPermissionAndSwitchToVideo();
259 return true;
260 } else {
261 return super.onOptionsItemSelected(item);
262 }
263 }
264
265 private void launchHelpInBrowser() {
266 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
267 try {
268 startActivity(intent);
269 } catch (final ActivityNotFoundException e) {
270 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
271 .show();
272 }
273 }
274
275 private void endCall(View view) {
276 endCall();
277 }
278
279 private void endCall() {
280 if (this.rtpConnectionReference == null) {
281 retractSessionProposal();
282 finish();
283 } else {
284 requireRtpConnection().endCall();
285 }
286 }
287
288 private void retractSessionProposal() {
289 final Intent intent = getIntent();
290 final String action = intent.getAction();
291 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
292 final Account account = extractAccount(intent);
293 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
294 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
295 if (!Intent.ACTION_VIEW.equals(action)
296 || state == null
297 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
298 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
299 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
300 }
301 xmppConnectionService
302 .getJingleConnectionManager()
303 .retractSessionProposal(account, with.asBareJid());
304 }
305
306 private void rejectCall(View view) {
307 requireRtpConnection().rejectCall();
308 finish();
309 }
310
311 private void acceptCall(View view) {
312 requestPermissionsAndAcceptCall();
313 }
314
315 private void acceptContentAdd() {
316 try {
317 requireRtpConnection()
318 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
319 } catch (final IllegalStateException e) {
320 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
321 }
322 }
323
324 private void requestPermissionAndSwitchToVideo() {
325 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
326 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
327 switchToVideo();
328 }
329 }
330
331 private void switchToVideo() {
332 try {
333 requireRtpConnection().addMedia(Media.VIDEO);
334 } catch (final IllegalStateException e) {
335 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
336 }
337 }
338
339 private void acceptContentAdd(final ContentAddition contentAddition) {
340 if (contentAddition == null
341 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
342 Log.d(Config.LOGTAG, "ignore press on content-accept button");
343 return;
344 }
345 requestPermissionAndAcceptContentAdd(contentAddition);
346 }
347
348 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
349 final List<String> permissions = permissions(contentAddition.media());
350 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
351 try {
352 requireRtpConnection().acceptContentAdd(contentAddition.summary);
353 } catch (final IllegalStateException e) {
354 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
355 }
356 }
357 }
358
359 private void rejectContentAdd(final View view) {
360 requireRtpConnection().rejectContentAdd();
361 }
362
363 private void requestPermissionsAndAcceptCall() {
364 final List<String> permissions = permissions(getMedia());
365 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
366 putScreenInCallMode();
367 checkRecorderAndAcceptCall();
368 }
369 }
370
371 private List<String> permissions(final Set<Media> media) {
372 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
373 if (media.contains(Media.VIDEO)) {
374 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
375 } else {
376 permissions.add(Manifest.permission.RECORD_AUDIO);
377 }
378 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
379 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
380 }
381 return permissions.build();
382 }
383
384 private void checkRecorderAndAcceptCall() {
385 checkMicrophoneAvailabilityAsync();
386 try {
387 requireRtpConnection().acceptCall();
388 } catch (final IllegalStateException e) {
389 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
390 }
391 }
392
393 private void checkMicrophoneAvailabilityAsync() {
394 new Thread(new MicrophoneAvailabilityCheck(this)).start();
395 }
396
397 private static class MicrophoneAvailabilityCheck implements Runnable {
398
399 private final WeakReference<Activity> activityReference;
400
401 private MicrophoneAvailabilityCheck(final Activity activity) {
402 this.activityReference = new WeakReference<>(activity);
403 }
404
405 @Override
406 public void run() {
407 final long start = SystemClock.elapsedRealtime();
408 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
409 final long stop = SystemClock.elapsedRealtime();
410 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
411 if (isMicrophoneAvailable) {
412 return;
413 }
414 final Activity activity = activityReference.get();
415 if (activity == null) {
416 return;
417 }
418 activity.runOnUiThread(
419 () ->
420 Toast.makeText(
421 activity,
422 R.string.microphone_unavailable,
423 Toast.LENGTH_LONG)
424 .show());
425 }
426 }
427
428 private void putScreenInCallMode() {
429 putScreenInCallMode(requireRtpConnection().getMedia());
430 }
431
432 private void putScreenInCallMode(final Set<Media> media) {
433 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
434 if (Media.audioOnly(media)) {
435 final JingleRtpConnection rtpConnection =
436 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
437 final CallIntegration callIntegration =
438 rtpConnection == null ? null : rtpConnection.getCallIntegration();
439 if (callIntegration == null
440 || callIntegration.getSelectedAudioDevice()
441 == CallIntegration.AudioDevice.EARPIECE) {
442 acquireProximityWakeLock();
443 }
444 }
445 lockOrientation(media);
446 }
447
448 private void lockOrientation(final Set<Media> media) {
449 if (Media.audioOnly(media)) {
450 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
451 } else {
452 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
453 }
454 }
455
456 @SuppressLint("WakelockTimeout")
457 private void acquireProximityWakeLock() {
458 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
459 if (powerManager == null) {
460 Log.e(Config.LOGTAG, "power manager not available");
461 return;
462 }
463 if (isFinishing()) {
464 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
465 return;
466 }
467 if (this.mProximityWakeLock == null) {
468 this.mProximityWakeLock =
469 powerManager.newWakeLock(
470 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
471 }
472 if (!this.mProximityWakeLock.isHeld()) {
473 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
474 this.mProximityWakeLock.acquire();
475 }
476 }
477
478 private void releaseProximityWakeLock() {
479 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
480 Log.d(Config.LOGTAG, "releasing proximity wake lock");
481 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
482 this.mProximityWakeLock = null;
483 }
484 }
485
486 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
487 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
488 acquireProximityWakeLock();
489 } else {
490 releaseProximityWakeLock();
491 }
492 }
493
494 @Override
495 protected void refreshUiReal() {}
496
497 @Override
498 public void onNewIntent(final Intent intent) {
499 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
500 super.onNewIntent(intent);
501 setIntent(intent);
502 if (xmppConnectionService == null) {
503 Log.d(
504 Config.LOGTAG,
505 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
506 return;
507 }
508 final Account account = extractAccount(intent);
509 final String action = intent.getAction();
510 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
511 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
512 if (sessionId != null) {
513 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
514 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
515 return;
516 }
517 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
518 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
519 requestPermissionsAndAcceptCall();
520 resetIntent(intent.getExtras());
521 }
522 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
523 proposeJingleRtpSession(account, with, actionToMedia(action));
524 setWith(account.getRoster().getContact(with), null);
525 } else {
526 throw new IllegalStateException("received onNewIntent without sessionId");
527 }
528 }
529
530 @Override
531 void onBackendConnected() {
532 final Intent intent = getIntent();
533 final String action = intent.getAction();
534 final Account account = extractAccount(intent);
535 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
536 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
537 if (sessionId != null) {
538 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
539 return;
540 }
541 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
542 Log.d(Config.LOGTAG, "intent action was accept");
543 requestPermissionsAndAcceptCall();
544 resetIntent(intent.getExtras());
545 }
546 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
547 proposeJingleRtpSession(account, with, actionToMedia(action));
548 setWith(account.getRoster().getContact(with), null);
549 } else if (Intent.ACTION_VIEW.equals(action)) {
550 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
551 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
552 final Set<Media> media = actionToMedia(lastAction);
553 final RtpEndUserState state =
554 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
555 if (state != null) {
556 Log.d(Config.LOGTAG, "restored last state from intent extra");
557 updateButtonConfiguration(state);
558 updateVerifiedShield(false);
559 updateStateDisplay(state);
560 updateIncomingCallScreen(state);
561 invalidateOptionsMenu();
562 }
563 setWith(account.getRoster().getContact(with), state);
564 if (xmppConnectionService
565 .getJingleConnectionManager()
566 .fireJingleRtpConnectionStateUpdates()) {
567 return;
568 }
569 if (END_CARD.contains(state)) {
570 return;
571 }
572 if (xmppConnectionService
573 .getJingleConnectionManager()
574 .hasMatchingProposal(account, with)) {
575 putScreenInCallMode(media);
576 return;
577 }
578 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
579 finish();
580 }
581 }
582
583 private void setWidth(final RtpEndUserState state) {
584 setWith(getWith(), state);
585 }
586
587 private void setWith(final Contact contact, final RtpEndUserState state) {
588 binding.with.setText(contact.getDisplayName());
589 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
590 .contains(state)) {
591 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
592 binding.withJid.setVisibility(View.VISIBLE);
593 } else {
594 binding.withJid.setVisibility(View.GONE);
595 }
596 }
597
598 private void proposeJingleRtpSession(
599 final Account account, final Jid with, final Set<Media> media) {
600 checkMicrophoneAvailabilityAsync();
601 if (with.isBareJid()) {
602 xmppConnectionService
603 .getJingleConnectionManager()
604 .proposeJingleRtpSession(account, with, media);
605 } else {
606 throw new IllegalStateException(
607 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
608 }
609 putScreenInCallMode(media);
610 }
611
612 @Override
613 public void onRequestPermissionsResult(
614 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
615 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
616 final PermissionUtils.PermissionResult permissionResult =
617 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
618 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
619 if (requestCode == REQUEST_ACCEPT_CALL) {
620 checkRecorderAndAcceptCall();
621 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
622 acceptContentAdd();
623 } else if (requestCode == REQUEST_ADD_CONTENT) {
624 switchToVideo();
625 }
626 } else {
627 @StringRes int res;
628 final String firstDenied =
629 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
630 if (firstDenied == null) {
631 return;
632 }
633 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
634 res = R.string.no_microphone_permission;
635 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
636 res = R.string.no_camera_permission;
637 } else {
638 throw new IllegalStateException("Invalid permission result request");
639 }
640 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
641 .show();
642 }
643 }
644
645 @Override
646 public void onStart() {
647 super.onStart();
648 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
649 this.binding.remoteVideo.setOnAspectRatioChanged(this);
650 }
651
652 @Override
653 public void onStop() {
654 mHandler.removeCallbacks(mTickExecutor);
655 binding.remoteVideo.release();
656 binding.remoteVideo.setOnAspectRatioChanged(null);
657 binding.localVideo.release();
658 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
659 final JingleRtpConnection jingleRtpConnection =
660 weakReference == null ? null : weakReference.get();
661 if (jingleRtpConnection != null) {
662 releaseVideoTracks(jingleRtpConnection);
663 }
664 releaseProximityWakeLock();
665 super.onStop();
666 }
667
668 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
669 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
670 if (remoteVideo.isPresent()) {
671 remoteVideo.get().removeSink(binding.remoteVideo);
672 }
673 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
674 if (localVideo.isPresent()) {
675 localVideo.get().removeSink(binding.localVideo);
676 }
677 }
678
679 @Override
680 public void onBackPressed() {
681 if (isConnected()) {
682 if (switchToPictureInPicture()) {
683 return;
684 }
685 } else {
686 endCall();
687 }
688 super.onBackPressed();
689 }
690
691 @Override
692 public void onUserLeaveHint() {
693 super.onUserLeaveHint();
694 if (switchToPictureInPicture()) {
695 return;
696 }
697 // TODO apparently this method is not getting called on Android 10 when using the task
698 // switcher
699 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
700 retractSessionProposal();
701 }
702 }
703
704 private boolean isConnected() {
705 final JingleRtpConnection connection =
706 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
707 final RtpEndUserState endUserState =
708 connection == null ? null : connection.getEndUserState();
709 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
710 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
711 }
712
713 private boolean switchToPictureInPicture() {
714 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
715 if (shouldBePictureInPicture()) {
716 startPictureInPicture();
717 return true;
718 }
719 }
720 return false;
721 }
722
723 @RequiresApi(api = Build.VERSION_CODES.O)
724 private void startPictureInPicture() {
725 try {
726 final Rational rational = this.binding.remoteVideo.getAspectRatio();
727 final Rational clippedRational = Rationals.clip(rational);
728 Log.d(
729 Config.LOGTAG,
730 "suggested rational " + rational + ". clipped to " + clippedRational);
731 enterPictureInPictureMode(
732 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
733 } catch (final IllegalStateException e) {
734 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
735 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
736 }
737 }
738
739 @Override
740 public void onAspectRatioChanged(final Rational rational) {
741 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
742 final Rational clippedRational = Rationals.clip(rational);
743 Log.d(
744 Config.LOGTAG,
745 "suggested rational after aspect ratio change "
746 + rational
747 + ". clipped to "
748 + clippedRational);
749 setPictureInPictureParams(
750 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
751 }
752 }
753
754 private boolean deviceSupportsPictureInPicture() {
755 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
756 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
757 } else {
758 return false;
759 }
760 }
761
762 private boolean shouldBePictureInPicture() {
763 try {
764 final JingleRtpConnection rtpConnection = requireRtpConnection();
765 return rtpConnection.getMedia().contains(Media.VIDEO)
766 && Arrays.asList(
767 RtpEndUserState.ACCEPTING_CALL,
768 RtpEndUserState.CONNECTING,
769 RtpEndUserState.CONNECTED)
770 .contains(rtpConnection.getEndUserState());
771 } catch (final IllegalStateException e) {
772 return false;
773 }
774 }
775
776 private boolean initializeActivityWithRunningRtpSession(
777 final Account account, Jid with, String sessionId) {
778 final WeakReference<JingleRtpConnection> reference =
779 xmppConnectionService
780 .getJingleConnectionManager()
781 .findJingleRtpConnection(account, with, sessionId);
782 if (reference == null || reference.get() == null) {
783 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
784 xmppConnectionService
785 .getJingleConnectionManager()
786 .getTerminalSessionState(with, sessionId);
787 if (terminatedRtpSession == null) {
788 throw new IllegalStateException(
789 "failed to initialize activity with running rtp session. session not found");
790 }
791 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
792 return true;
793 }
794 this.rtpConnectionReference = reference;
795 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
796 final boolean verified = requireRtpConnection().isVerified();
797 if (currentState == RtpEndUserState.ENDED) {
798 finish();
799 return true;
800 }
801 final Set<Media> media = getMedia();
802 final ContentAddition contentAddition = getPendingContentAddition();
803 if (currentState == RtpEndUserState.INCOMING_CALL) {
804 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
805 }
806 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
807 requireRtpConnection().getState())) {
808 putScreenInCallMode();
809 }
810 setWidth(currentState);
811 updateVideoViews(currentState);
812 updateStateDisplay(currentState, media, contentAddition);
813 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
814 updateButtonConfiguration(currentState, media, contentAddition);
815 updateIncomingCallScreen(currentState);
816 invalidateOptionsMenu();
817 return false;
818 }
819
820 private void initializeWithTerminatedSessionState(
821 final Account account,
822 final Jid with,
823 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
824 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
825 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
826 finish();
827 return;
828 }
829 final RtpEndUserState state = terminatedRtpSession.state;
830 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
831 updateButtonConfiguration(state);
832 updateStateDisplay(state);
833 updateIncomingCallScreen(state);
834 updateCallDuration();
835 updateVerifiedShield(false);
836 invalidateOptionsMenu();
837 setWith(account.getRoster().getContact(with), state);
838 }
839
840 private void reInitializeActivityWithRunningRtpSession(
841 final Account account, Jid with, String sessionId) {
842 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
843 resetIntent(account, with, sessionId);
844 }
845
846 private void resetIntent(final Account account, final Jid with, final String sessionId) {
847 final Intent intent = new Intent(Intent.ACTION_VIEW);
848 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
849 intent.putExtra(EXTRA_WITH, with.toEscapedString());
850 intent.putExtra(EXTRA_SESSION_ID, sessionId);
851 setIntent(intent);
852 }
853
854 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
855 surfaceViewRenderer.setVisibility(View.VISIBLE);
856 try {
857 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
858 } catch (final IllegalStateException ignored) {
859 // SurfaceViewRenderer was already initialized
860 } catch (final RuntimeException e) {
861 if (Throwables.getRootCause(e) instanceof GLException glException) {
862 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
863 }
864 }
865 surfaceViewRenderer.setEnableHardwareScaler(true);
866 }
867
868 private void updateStateDisplay(final RtpEndUserState state) {
869 updateStateDisplay(state, Collections.emptySet(), null);
870 }
871
872 private void updateStateDisplay(
873 final RtpEndUserState state,
874 final Set<Media> media,
875 final ContentAddition contentAddition) {
876 switch (state) {
877 case INCOMING_CALL -> {
878 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
879 if (media.contains(Media.VIDEO)) {
880 setTitle(R.string.rtp_state_incoming_video_call);
881 } else {
882 setTitle(R.string.rtp_state_incoming_call);
883 }
884 }
885 case INCOMING_CONTENT_ADD -> {
886 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
887 setTitle(R.string.rtp_state_content_add_video);
888 } else {
889 setTitle(R.string.rtp_state_content_add);
890 }
891 }
892 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
893 case CONNECTED -> setTitle(R.string.rtp_state_connected);
894 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
895 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
896 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
897 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
898 case RINGING -> setTitle(R.string.rtp_state_ringing);
899 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
900 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
901 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
902 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
903 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
904 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
905 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
906 case ENDED -> throw new IllegalStateException(
907 "Activity should have called finishAndReleaseWakeLock();");
908 default -> throw new IllegalStateException(
909 String.format("State %s has not been handled in UI", state));
910 }
911 }
912
913 private void updateVerifiedShield(final boolean verified) {
914 if (isPictureInPicture()) {
915 this.binding.verified.setVisibility(View.GONE);
916 return;
917 }
918 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
919 }
920
921 private void updateIncomingCallScreen(final RtpEndUserState state) {
922 updateIncomingCallScreen(state, null);
923 }
924
925 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
926 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
927 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
928 if (show) {
929 binding.contactPhoto.setVisibility(View.VISIBLE);
930 if (contact == null) {
931 AvatarWorkerTask.loadAvatar(
932 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
933 } else {
934 AvatarWorkerTask.loadAvatar(
935 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
936 }
937 } else {
938 binding.contactPhoto.setVisibility(View.GONE);
939 }
940 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
941 binding.usingAccount.setVisibility(View.VISIBLE);
942 binding.usingAccount.setText(
943 getString(
944 R.string.using_account,
945 account.getJid().asBareJid().toEscapedString()));
946 } else {
947 binding.usingAccount.setVisibility(View.GONE);
948 binding.contactPhoto.setVisibility(View.GONE);
949 }
950 }
951
952 private Set<Media> getMedia() {
953 return requireRtpConnection().getMedia();
954 }
955
956 public ContentAddition getPendingContentAddition() {
957 return requireRtpConnection().getPendingContentAddition();
958 }
959
960 private void updateButtonConfiguration(final RtpEndUserState state) {
961 updateButtonConfiguration(state, Collections.emptySet(), null);
962 }
963
964 @SuppressLint("RestrictedApi")
965 private void updateButtonConfiguration(
966 final RtpEndUserState state,
967 final Set<Media> media,
968 final ContentAddition contentAddition) {
969 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
970 this.binding.rejectCall.setVisibility(View.INVISIBLE);
971 this.binding.endCall.setVisibility(View.INVISIBLE);
972 this.binding.acceptCall.setVisibility(View.INVISIBLE);
973 } else if (state == RtpEndUserState.INCOMING_CALL) {
974 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
975 this.binding.rejectCall.setOnClickListener(this::rejectCall);
976 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
977 this.binding.rejectCall.setVisibility(View.VISIBLE);
978 this.binding.endCall.setVisibility(View.INVISIBLE);
979 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
980 this.binding.acceptCall.setOnClickListener(this::acceptCall);
981 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
982 this.binding.acceptCall.setVisibility(View.VISIBLE);
983 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
984 this.binding.rejectCall.setContentDescription(
985 getString(R.string.reject_switch_to_video));
986 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
987 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
988 this.binding.rejectCall.setVisibility(View.VISIBLE);
989 this.binding.endCall.setVisibility(View.INVISIBLE);
990 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
991 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
992 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
993 this.binding.acceptCall.setVisibility(View.VISIBLE);
994 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
995 .contains(state)) {
996 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
997 this.binding.rejectCall.setOnClickListener(this::exit);
998 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
999 this.binding.rejectCall.setVisibility(View.VISIBLE);
1000 this.binding.endCall.setVisibility(View.INVISIBLE);
1001 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1002 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1003 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
1004 this.binding.acceptCall.setVisibility(View.VISIBLE);
1005 } else if (asList(
1006 RtpEndUserState.CONNECTIVITY_ERROR,
1007 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1008 RtpEndUserState.APPLICATION_ERROR,
1009 RtpEndUserState.RETRACTED,
1010 RtpEndUserState.SECURITY_ERROR)
1011 .contains(state)) {
1012 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1013 this.binding.rejectCall.setOnClickListener(this::exit);
1014 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1015 this.binding.rejectCall.setVisibility(View.VISIBLE);
1016 this.binding.endCall.setVisibility(View.INVISIBLE);
1017 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1018 this.binding.acceptCall.setOnClickListener(this::retry);
1019 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1020 this.binding.acceptCall.setVisibility(View.VISIBLE);
1021 } else {
1022 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1023 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1024 this.binding.endCall.setOnClickListener(this::endCall);
1025 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1026 this.binding.endCall.setVisibility(View.VISIBLE);
1027 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1028 }
1029 updateInCallButtonConfiguration(state, media);
1030 }
1031
1032 private boolean isPictureInPicture() {
1033 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1034 return isInPictureInPictureMode();
1035 } else {
1036 return false;
1037 }
1038 }
1039
1040 private void updateInCallButtonConfiguration() {
1041 updateInCallButtonConfiguration(
1042 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1043 }
1044
1045 @SuppressLint("RestrictedApi")
1046 private void updateInCallButtonConfiguration(
1047 final RtpEndUserState state, final Set<Media> media) {
1048 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1049 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1050 if (media.contains(Media.VIDEO)) {
1051 final JingleRtpConnection rtpConnection = requireRtpConnection();
1052 updateInCallButtonConfigurationVideo(
1053 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1054 } else {
1055 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1056 updateInCallButtonConfigurationSpeaker(
1057 callIntegration.getSelectedAudioDevice(),
1058 callIntegration.getAudioDevices().size());
1059 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1060 }
1061 if (media.contains(Media.AUDIO)) {
1062 updateInCallButtonConfigurationMicrophone(
1063 requireRtpConnection().isMicrophoneEnabled());
1064 } else {
1065 this.binding.inCallActionLeft.setVisibility(View.GONE);
1066 }
1067 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1068 && !isPictureInPicture()
1069 && Media.audioOnly(media)) {
1070 final CallIntegration callIntegration = requireCallIntegration();
1071 updateInCallButtonConfigurationSpeaker(
1072 callIntegration.getSelectedAudioDevice(),
1073 callIntegration.getAudioDevices().size());
1074 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1075 } else {
1076 this.binding.inCallActionLeft.setVisibility(View.GONE);
1077 this.binding.inCallActionRight.setVisibility(View.GONE);
1078 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1079 }
1080 }
1081
1082 @SuppressLint("RestrictedApi")
1083 private void updateInCallButtonConfigurationSpeaker(
1084 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1085 switch (selectedAudioDevice) {
1086 case EARPIECE -> {
1087 this.binding.inCallActionRight.setImageResource(
1088 R.drawable.ic_volume_off_black_24dp);
1089 if (numberOfChoices >= 2) {
1090 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1091 } else {
1092 this.binding.inCallActionRight.setOnClickListener(null);
1093 this.binding.inCallActionRight.setClickable(false);
1094 }
1095 }
1096 case WIRED_HEADSET -> {
1097 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1098 this.binding.inCallActionRight.setOnClickListener(null);
1099 this.binding.inCallActionRight.setClickable(false);
1100 }
1101 case SPEAKER_PHONE -> {
1102 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1103 if (numberOfChoices >= 2) {
1104 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1105 } else {
1106 this.binding.inCallActionRight.setOnClickListener(null);
1107 this.binding.inCallActionRight.setClickable(false);
1108 }
1109 }
1110 case BLUETOOTH -> {
1111 this.binding.inCallActionRight.setImageResource(
1112 R.drawable.ic_bluetooth_audio_black_24dp);
1113 this.binding.inCallActionRight.setOnClickListener(null);
1114 this.binding.inCallActionRight.setClickable(false);
1115 }
1116 }
1117 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1118 }
1119
1120 @SuppressLint("RestrictedApi")
1121 private void updateInCallButtonConfigurationVideo(
1122 final boolean videoEnabled, final boolean isCameraSwitchable) {
1123 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1124 if (isCameraSwitchable) {
1125 this.binding.inCallActionFarRight.setImageResource(
1126 R.drawable.ic_flip_camera_android_black_24dp);
1127 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1128 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1129 } else {
1130 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1131 }
1132 if (videoEnabled) {
1133 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1134 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1135 } else {
1136 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1137 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1138 }
1139 }
1140
1141 private void switchCamera(final View view) {
1142 Futures.addCallback(
1143 requireRtpConnection().switchCamera(),
1144 new FutureCallback<>() {
1145 @Override
1146 public void onSuccess(@Nullable Boolean isFrontCamera) {
1147 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1148 }
1149
1150 @Override
1151 public void onFailure(@NonNull final Throwable throwable) {
1152 Log.d(
1153 Config.LOGTAG,
1154 "could not switch camera",
1155 Throwables.getRootCause(throwable));
1156 Toast.makeText(
1157 RtpSessionActivity.this,
1158 R.string.could_not_switch_camera,
1159 Toast.LENGTH_LONG)
1160 .show();
1161 }
1162 },
1163 MainThreadExecutor.getInstance());
1164 }
1165
1166 private void enableVideo(View view) {
1167 try {
1168 requireRtpConnection().setVideoEnabled(true);
1169 } catch (final IllegalStateException e) {
1170 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1171 return;
1172 }
1173 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1174 }
1175
1176 private void disableVideo(View view) {
1177 final JingleRtpConnection rtpConnection = requireRtpConnection();
1178 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1179 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1180 rtpConnection.retractContentAdd();
1181 return;
1182 }
1183 requireRtpConnection().setVideoEnabled(false);
1184 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1185 }
1186
1187 @SuppressLint("RestrictedApi")
1188 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1189 if (microphoneEnabled) {
1190 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1191 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1192 } else {
1193 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1194 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1195 }
1196 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1197 }
1198
1199 private void updateCallDuration() {
1200 final JingleRtpConnection connection =
1201 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1202 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1203 this.binding.duration.setVisibility(View.GONE);
1204 return;
1205 }
1206 if (connection.zeroDuration()) {
1207 this.binding.duration.setVisibility(View.GONE);
1208 } else {
1209 this.binding.duration.setText(
1210 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1211 this.binding.duration.setVisibility(View.VISIBLE);
1212 }
1213 }
1214
1215 private void updateVideoViews(final RtpEndUserState state) {
1216 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1217 binding.localVideo.setVisibility(View.GONE);
1218 binding.localVideo.release();
1219 binding.remoteVideoWrapper.setVisibility(View.GONE);
1220 binding.remoteVideo.release();
1221 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1222 if (isPictureInPicture()) {
1223 binding.appBarLayout.setVisibility(View.GONE);
1224 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1225 if (Arrays.asList(
1226 RtpEndUserState.APPLICATION_ERROR,
1227 RtpEndUserState.CONNECTIVITY_ERROR,
1228 RtpEndUserState.SECURITY_ERROR)
1229 .contains(state)) {
1230 binding.pipWarning.setVisibility(View.VISIBLE);
1231 binding.pipWaiting.setVisibility(View.GONE);
1232 } else {
1233 binding.pipWarning.setVisibility(View.GONE);
1234 binding.pipWaiting.setVisibility(View.GONE);
1235 }
1236 } else {
1237 binding.appBarLayout.setVisibility(View.VISIBLE);
1238 binding.pipPlaceholder.setVisibility(View.GONE);
1239 }
1240 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1241 return;
1242 }
1243 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1244 binding.localVideo.setVisibility(View.GONE);
1245 binding.remoteVideoWrapper.setVisibility(View.GONE);
1246 binding.appBarLayout.setVisibility(View.GONE);
1247 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1248 binding.pipWarning.setVisibility(View.GONE);
1249 binding.pipWaiting.setVisibility(View.VISIBLE);
1250 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1251 return;
1252 }
1253 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1254 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1255 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1256 // paint local view over remote view
1257 binding.localVideo.setZOrderMediaOverlay(true);
1258 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1259 addSink(localVideoTrack.get(), binding.localVideo);
1260 } else {
1261 binding.localVideo.setVisibility(View.GONE);
1262 }
1263 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1264 if (remoteVideoTrack.isPresent()) {
1265 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1266 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1267 binding.remoteVideo.setScalingType(
1268 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1269 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1270 if (state == RtpEndUserState.CONNECTED) {
1271 binding.appBarLayout.setVisibility(View.GONE);
1272 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1273 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1274 } else {
1275 binding.appBarLayout.setVisibility(View.VISIBLE);
1276 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1277 binding.remoteVideoWrapper.setVisibility(View.GONE);
1278 }
1279 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1280 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1281 } else {
1282 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1283 }
1284 } else {
1285 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1286 binding.remoteVideoWrapper.setVisibility(View.GONE);
1287 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1288 }
1289 }
1290
1291 private Optional<VideoTrack> getLocalVideoTrack() {
1292 final JingleRtpConnection connection =
1293 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1294 if (connection == null) {
1295 return Optional.absent();
1296 }
1297 return connection.getLocalVideoTrack();
1298 }
1299
1300 private Optional<VideoTrack> getRemoteVideoTrack() {
1301 final JingleRtpConnection connection =
1302 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1303 if (connection == null) {
1304 return Optional.absent();
1305 }
1306 return connection.getRemoteVideoTrack();
1307 }
1308
1309 private void disableMicrophone(View view) {
1310 final JingleRtpConnection rtpConnection = requireRtpConnection();
1311 if (rtpConnection.setMicrophoneEnabled(false)) {
1312 updateInCallButtonConfiguration();
1313 }
1314 }
1315
1316 private void enableMicrophone(View view) {
1317 final JingleRtpConnection rtpConnection = requireRtpConnection();
1318 if (rtpConnection.setMicrophoneEnabled(true)) {
1319 updateInCallButtonConfiguration();
1320 }
1321 }
1322
1323 private void switchToEarpiece(final View view) {
1324 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1325 acquireProximityWakeLock();
1326 }
1327
1328 private void switchToSpeaker(final View view) {
1329 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1330 releaseProximityWakeLock();
1331 }
1332
1333 private void retry(final View view) {
1334 final Intent intent = getIntent();
1335 final Account account = extractAccount(intent);
1336 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1337 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1338 final String action = intent.getAction();
1339 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1340 this.rtpConnectionReference = null;
1341 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1342 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1343 }
1344
1345 private void exit(final View view) {
1346 finish();
1347 }
1348
1349 private void recordVoiceMail(final View view) {
1350 final Intent intent = getIntent();
1351 final Account account = extractAccount(intent);
1352 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1353 final Conversation conversation =
1354 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1355 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1356 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1357 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1358 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1359 launchIntent.putExtra(
1360 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1361 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1362 startActivity(launchIntent);
1363 finish();
1364 }
1365
1366 private Contact getWith() {
1367 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1368 final Account account = id.account;
1369 return account.getRoster().getContact(id.with);
1370 }
1371
1372 private JingleRtpConnection requireRtpConnection() {
1373 final JingleRtpConnection connection =
1374 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1375 if (connection == null) {
1376 throw new IllegalStateException("No RTP connection found");
1377 }
1378 return connection;
1379 }
1380
1381 private CallIntegration requireCallIntegration() {
1382 return requireOngoingRtpSession().getCallIntegration();
1383 }
1384
1385 private OngoingRtpSession requireOngoingRtpSession() {
1386 final JingleRtpConnection connection =
1387 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1388 if (connection != null) {
1389 return connection;
1390 }
1391 final Intent currentIntent = getIntent();
1392 final String withExtra =
1393 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1394 final var account = extractAccount(currentIntent);
1395 if (withExtra == null) {
1396 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1397 }
1398 final var matching =
1399 xmppConnectionService
1400 .getJingleConnectionManager()
1401 .matchingProposal(account, Jid.of(withExtra));
1402 if (matching.isPresent()) {
1403 return matching.get();
1404 }
1405 throw new IllegalStateException("No matching session proposal");
1406 }
1407
1408 @Override
1409 public void onJingleRtpConnectionUpdate(
1410 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1411 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1412 if (END_CARD.contains(state)) {
1413 Log.d(Config.LOGTAG, "end card reached");
1414 releaseProximityWakeLock();
1415 runOnUiThread(
1416 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1417 }
1418 if (with.isBareJid()) {
1419 updateRtpSessionProposalState(account, with, state);
1420 return;
1421 }
1422 if (emptyReference(this.rtpConnectionReference)) {
1423 if (END_CARD.contains(state)) {
1424 Log.d(Config.LOGTAG, "not reinitializing session");
1425 return;
1426 }
1427 // this happens when going from proposed session to actual session
1428 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1429 return;
1430 }
1431 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1432 final boolean verified = requireRtpConnection().isVerified();
1433 final Set<Media> media = getMedia();
1434 lockOrientation(media);
1435 final ContentAddition contentAddition = getPendingContentAddition();
1436 final Contact contact = getWith();
1437 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1438 if (state == RtpEndUserState.ENDED) {
1439 finish();
1440 return;
1441 }
1442 runOnUiThread(
1443 () -> {
1444 updateStateDisplay(state, media, contentAddition);
1445 updateVerifiedShield(
1446 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1447 updateButtonConfiguration(state, media, contentAddition);
1448 updateVideoViews(state);
1449 updateIncomingCallScreen(state, contact);
1450 invalidateOptionsMenu();
1451 });
1452 if (END_CARD.contains(state)) {
1453 final JingleRtpConnection rtpConnection = requireRtpConnection();
1454 resetIntent(account, with, state, rtpConnection.getMedia());
1455 releaseVideoTracks(rtpConnection);
1456 this.rtpConnectionReference = null;
1457 }
1458 } else {
1459 Log.d(Config.LOGTAG, "received update for other rtp session");
1460 }
1461 }
1462
1463 @Override
1464 public void onAudioDeviceChanged(
1465 final CallIntegration.AudioDevice selectedAudioDevice,
1466 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1467 Log.d(
1468 Config.LOGTAG,
1469 "onAudioDeviceChanged in activity: selected:"
1470 + selectedAudioDevice
1471 + ", available:"
1472 + availableAudioDevices);
1473 try {
1474 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1475 final RtpEndUserState endUserState;
1476 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1477 endUserState = jingleRtpConnection.getEndUserState();
1478 } else {
1479 // for session proposals all end user states are functionally the same
1480 endUserState = RtpEndUserState.RINGING;
1481 }
1482 final Set<Media> media = ongoingRtpSession.getMedia();
1483 if (END_CARD.contains(endUserState)) {
1484 Log.d(
1485 Config.LOGTAG,
1486 "onAudioDeviceChanged() nothing to do because end card has been reached");
1487 } else {
1488 if (Media.audioOnly(media)
1489 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1490 final CallIntegration callIntegration = requireCallIntegration();
1491 updateInCallButtonConfigurationSpeaker(
1492 callIntegration.getSelectedAudioDevice(),
1493 callIntegration.getAudioDevices().size());
1494 }
1495 Log.d(
1496 Config.LOGTAG,
1497 "put proximity wake lock into proper state after device update");
1498 putProximityWakeLockInProperState(selectedAudioDevice);
1499 }
1500 } catch (final IllegalStateException e) {
1501 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1502 }
1503 }
1504
1505 private void updateRtpSessionProposalState(
1506 final Account account, final Jid with, final RtpEndUserState state) {
1507 final Intent currentIntent = getIntent();
1508 final String withExtra =
1509 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1510 if (withExtra == null) {
1511 return;
1512 }
1513 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1514 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1515 runOnUiThread(
1516 () -> {
1517 updateVerifiedShield(false);
1518 updateStateDisplay(state);
1519 updateButtonConfiguration(state, media, null);
1520 updateIncomingCallScreen(state);
1521 invalidateOptionsMenu();
1522 });
1523 resetIntent(account, with, state, media);
1524 }
1525 }
1526
1527 private void resetIntent(final Bundle extras) {
1528 final Intent intent = new Intent(Intent.ACTION_VIEW);
1529 intent.putExtras(extras);
1530 setIntent(intent);
1531 }
1532
1533 private void resetIntent(
1534 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1535 final Intent intent = new Intent(Intent.ACTION_VIEW);
1536 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1537 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1538 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1539 } else {
1540 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1541 }
1542 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1543 intent.putExtra(
1544 EXTRA_LAST_ACTION,
1545 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1546 setIntent(intent);
1547 }
1548
1549 private static boolean emptyReference(final WeakReference<?> weakReference) {
1550 return weakReference == null || weakReference.get() == null;
1551 }
1552}