1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.android.material.floatingactionbutton.FloatingActionButton;
36import com.google.common.base.Optional;
37import com.google.common.base.Preconditions;
38import com.google.common.base.Strings;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.CallIntegration;
52import eu.siacs.conversations.services.CallIntegrationConnectionService;
53import eu.siacs.conversations.services.XmppConnectionService;
54import eu.siacs.conversations.ui.util.AvatarWorkerTask;
55import eu.siacs.conversations.ui.util.MainThreadExecutor;
56import eu.siacs.conversations.ui.util.Rationals;
57import eu.siacs.conversations.utils.PermissionUtils;
58import eu.siacs.conversations.utils.TimeFrameUtils;
59import eu.siacs.conversations.xmpp.Jid;
60import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
61import eu.siacs.conversations.xmpp.jingle.ContentAddition;
62import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
63import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
64import eu.siacs.conversations.xmpp.jingle.Media;
65import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
66import eu.siacs.conversations.xmpp.jingle.RtpCapability;
67import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
68
69import org.webrtc.RendererCommon;
70import org.webrtc.SurfaceViewRenderer;
71import org.webrtc.VideoTrack;
72
73import java.lang.ref.WeakReference;
74import java.util.Arrays;
75import java.util.Collections;
76import java.util.List;
77import java.util.Set;
78
79public class RtpSessionActivity extends XmppActivity
80 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
81 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
82
83 public static final String EXTRA_WITH = "with";
84 public static final String EXTRA_SESSION_ID = "session_id";
85 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
86 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
87 public static final String EXTRA_LAST_ACTION = "last_action";
88 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
89 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
90 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
91
92 private static final int CALL_DURATION_UPDATE_INTERVAL = 250;
93 private static final int BUTTON_VISIBILITY_TIMEOUT = 10_000;
94
95 private static final List<RtpEndUserState> END_CARD =
96 Arrays.asList(
97 RtpEndUserState.APPLICATION_ERROR,
98 RtpEndUserState.SECURITY_ERROR,
99 RtpEndUserState.DECLINED_OR_BUSY,
100 RtpEndUserState.CONTACT_OFFLINE,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
103 RtpEndUserState.RETRACTED);
104 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
105 Arrays.asList(
106 RtpEndUserState.APPLICATION_ERROR,
107 RtpEndUserState.CONNECTIVITY_ERROR,
108 RtpEndUserState.SECURITY_ERROR);
109 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
110 Arrays.asList(
111 RtpEndUserState.CONNECTING,
112 RtpEndUserState.CONNECTED,
113 RtpEndUserState.RECONNECTING,
114 RtpEndUserState.INCOMING_CONTENT_ADD);
115 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
116 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
117 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
118 Arrays.asList(
119 RtpEndUserState.ACCEPTING_CALL,
120 RtpEndUserState.CONNECTING,
121 RtpEndUserState.RECONNECTING);
122 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
123 new ImmutableList.Builder<RtpEndUserState>()
124 .add(RtpEndUserState.FINDING_DEVICE)
125 .add(RtpEndUserState.RINGING)
126 .add(RtpEndUserState.ACCEPTING_CALL)
127 .add(RtpEndUserState.CONNECTING)
128 .addAll(STATES_CONSIDERED_CONNECTED)
129 .build();
130 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
131 private static final int REQUEST_ACCEPT_CALL = 0x1111;
132 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
133 private static final int REQUEST_ADD_CONTENT = 0x1113;
134 private WeakReference<JingleRtpConnection> rtpConnectionReference;
135
136 private ActivityRtpSessionBinding binding;
137 private PowerManager.WakeLock mProximityWakeLock;
138
139 private final Handler mHandler = new Handler();
140 private final Runnable mTickExecutor =
141 new Runnable() {
142 @Override
143 public void run() {
144 updateCallDuration();
145 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
146 }
147 };
148 private boolean buttonsHiddenAfterTimeout = false;
149 private final Runnable mVisibilityToggleExecutor = this::updateButtonInVideoCallVisibility;
150
151 public static Set<Media> actionToMedia(final String action) {
152 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
153 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
154 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
155 return ImmutableSet.of(Media.AUDIO);
156 } else {
157 Log.w(
158 Config.LOGTAG,
159 "actionToMedia can not get media set from unknown action " + action);
160 return Collections.emptySet();
161 }
162 }
163
164 private static void addSink(
165 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
166 try {
167 videoTrack.addSink(surfaceViewRenderer);
168 } catch (final IllegalStateException e) {
169 Log.e(
170 Config.LOGTAG,
171 "possible race condition on trying to display video track. ignoring",
172 e);
173 }
174 }
175
176 @Override
177 public void onCreate(Bundle savedInstanceState) {
178 super.onCreate(savedInstanceState);
179 getWindow()
180 .addFlags(
181 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
182 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
183 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
184 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
185 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
186 this.binding.remoteVideo.setOnClickListener(this::onVideoScreenClick);
187 this.binding.localVideo.setOnClickListener(this::onVideoScreenClick);
188 setSupportActionBar(binding.toolbar);
189 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
190 }
191
192 private void onVideoScreenClick(final View view) {
193 resetVisibilityExecutorShowButtons();
194 }
195
196 @Override
197 public boolean onCreateOptionsMenu(final Menu menu) {
198 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
199 final MenuItem help = menu.findItem(R.id.action_help);
200 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
201 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
202 help.setVisible(Config.HELP != null && isHelpButtonVisible());
203 gotoChat.setVisible(isSwitchToConversationVisible());
204 switchToVideo.setVisible(isSwitchToVideoVisible());
205 return super.onCreateOptionsMenu(menu);
206 }
207
208 @Override
209 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
210 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
211 if (xmppConnectionService != null) {
212 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
213 return true;
214 }
215 }
216 }
217 return super.onKeyDown(keyCode, event);
218 }
219
220 private boolean isHelpButtonVisible() {
221 try {
222 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
223 } catch (IllegalStateException e) {
224 final Intent intent = getIntent();
225 final String state =
226 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
227 if (state != null) {
228 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
229 } else {
230 return false;
231 }
232 }
233 }
234
235 private boolean isSwitchToConversationVisible() {
236 final JingleRtpConnection connection =
237 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
238 return connection != null
239 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
240 }
241
242 private boolean isSwitchToVideoVisible() {
243 final JingleRtpConnection connection =
244 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
245 if (connection == null) {
246 return false;
247 }
248 return connection.isSwitchToVideoAvailable();
249 }
250
251 private void switchToConversation() {
252 final Contact contact = getWith();
253 final Conversation conversation =
254 xmppConnectionService.findOrCreateConversation(
255 contact.getAccount(), contact.getJid(), false, true);
256 switchToConversation(conversation);
257 }
258
259 public boolean onOptionsItemSelected(final MenuItem item) {
260 final var itemItem = item.getItemId();
261 if (itemItem == R.id.action_help) {
262 launchHelpInBrowser();
263 return true;
264 } else if (itemItem == R.id.action_goto_chat) {
265 switchToConversation();
266 return true;
267 } else if (itemItem == R.id.action_switch_to_video) {
268 requestPermissionAndSwitchToVideo();
269 return true;
270 } else {
271 return super.onOptionsItemSelected(item);
272 }
273 }
274
275 private void launchHelpInBrowser() {
276 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
277 try {
278 startActivity(intent);
279 } catch (final ActivityNotFoundException e) {
280 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
281 .show();
282 }
283 }
284
285 private void endCall(View view) {
286 endCall();
287 }
288
289 private void endCall() {
290 if (this.rtpConnectionReference == null) {
291 retractSessionProposal();
292 finish();
293 } else {
294 requireRtpConnection().endCall();
295 }
296 }
297
298 private void retractSessionProposal() {
299 final Intent intent = getIntent();
300 final String action = intent.getAction();
301 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
302 final Account account = extractAccount(intent);
303 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
304 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
305 if (!Intent.ACTION_VIEW.equals(action)
306 || state == null
307 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
308 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
309 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
310 }
311 xmppConnectionService
312 .getJingleConnectionManager()
313 .retractSessionProposal(account, with.asBareJid());
314 }
315
316 private void rejectCall(View view) {
317 requireRtpConnection().rejectCall();
318 finish();
319 }
320
321 private void acceptCall(View view) {
322 requestPermissionsAndAcceptCall();
323 }
324
325 private void acceptContentAdd() {
326 try {
327 final ContentAddition pendingContentAddition =
328 requireRtpConnection().getPendingContentAddition();
329 if (pendingContentAddition == null) {
330 Log.d(Config.LOGTAG, "content offer was gone after granting permission");
331 return;
332 }
333 requireRtpConnection().acceptContentAdd(pendingContentAddition.summary);
334 } catch (final IllegalStateException e) {
335 Toast.makeText(this, Strings.nullToEmpty(e.getMessage()), Toast.LENGTH_SHORT).show();
336 }
337 }
338
339 private void requestPermissionAndSwitchToVideo() {
340 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
341 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
342 switchToVideo();
343 }
344 }
345
346 private void switchToVideo() {
347 try {
348 requireRtpConnection().addMedia(Media.VIDEO);
349 } catch (final IllegalStateException e) {
350 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
351 }
352 }
353
354 private void acceptContentAdd(final ContentAddition contentAddition) {
355 if (contentAddition == null
356 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
357 Log.d(Config.LOGTAG, "ignore press on content-accept button");
358 return;
359 }
360 requestPermissionAndAcceptContentAdd(contentAddition);
361 }
362
363 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
364 final List<String> permissions = permissions(contentAddition.media());
365 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
366 try {
367 requireRtpConnection().acceptContentAdd(contentAddition.summary);
368 } catch (final IllegalStateException e) {
369 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
370 }
371 }
372 }
373
374 private void rejectContentAdd(final View view) {
375 requireRtpConnection().rejectContentAdd();
376 }
377
378 private void requestPermissionsAndAcceptCall() {
379 final List<String> permissions = permissions(getMedia());
380 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
381 putScreenInCallMode();
382 acceptCall();
383 }
384 }
385
386 private List<String> permissions(final Set<Media> media) {
387 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
388 if (media.contains(Media.VIDEO)) {
389 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
390 } else {
391 permissions.add(Manifest.permission.RECORD_AUDIO);
392 }
393 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
394 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
395 }
396 return permissions.build();
397 }
398
399 private void acceptCall() {
400 try {
401 requireRtpConnection().acceptCall();
402 } catch (final IllegalStateException e) {
403 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
404 }
405 }
406
407 private void putScreenInCallMode() {
408 putScreenInCallMode(requireRtpConnection().getMedia());
409 }
410
411 private void putScreenInCallMode(final Set<Media> media) {
412 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
413 if (Media.audioOnly(media)) {
414 final JingleRtpConnection rtpConnection =
415 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
416 final CallIntegration callIntegration =
417 rtpConnection == null ? null : rtpConnection.getCallIntegration();
418 if (callIntegration == null
419 || callIntegration.getSelectedAudioDevice()
420 == CallIntegration.AudioDevice.EARPIECE) {
421 acquireProximityWakeLock();
422 }
423 }
424 lockOrientation(media);
425 }
426
427 private void lockOrientation(final Set<Media> media) {
428 if (Media.audioOnly(media)) {
429 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
430 } else {
431 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
432 }
433 }
434
435 @SuppressLint("WakelockTimeout")
436 private void acquireProximityWakeLock() {
437 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
438 if (powerManager == null) {
439 Log.e(Config.LOGTAG, "power manager not available");
440 return;
441 }
442 if (isFinishing()) {
443 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
444 return;
445 }
446 if (this.mProximityWakeLock == null) {
447 this.mProximityWakeLock =
448 powerManager.newWakeLock(
449 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
450 }
451 if (!this.mProximityWakeLock.isHeld()) {
452 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
453 this.mProximityWakeLock.acquire();
454 }
455 }
456
457 private void releaseProximityWakeLock() {
458 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
459 Log.d(Config.LOGTAG, "releasing proximity wake lock");
460 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
461 this.mProximityWakeLock = null;
462 }
463 }
464
465 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
466 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
467 acquireProximityWakeLock();
468 } else {
469 releaseProximityWakeLock();
470 }
471 }
472
473 @Override
474 protected void refreshUiReal() {}
475
476 @Override
477 public void onNewIntent(final Intent intent) {
478 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
479 super.onNewIntent(intent);
480 if (intent == null) {
481 return;
482 }
483 setIntent(intent);
484 if (xmppConnectionService == null) {
485 Log.d(
486 Config.LOGTAG,
487 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
488 return;
489 }
490 initializeWithIntent(Event.ON_NEW_INTENT, intent);
491 }
492
493 @Override
494 protected void onBackendConnected() {
495 final var intent = getIntent();
496 if (intent == null) {
497 return;
498 }
499 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
500 }
501
502 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
503 final String action = intent.getAction();
504 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
505 final Account account = extractAccount(intent);
506 final var extraWith = intent.getStringExtra(EXTRA_WITH);
507 final Jid with = Strings.isNullOrEmpty(extraWith) ? null : Jid.ofEscaped(extraWith);
508 if (with == null || account == null) {
509 Log.e(Config.LOGTAG, "intent is missing extras (account or with)");
510 return;
511 }
512 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
513 if (sessionId != null) {
514 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
515 return;
516 }
517 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
518 Log.d(Config.LOGTAG, "intent action was accept");
519 requestPermissionsAndAcceptCall();
520 resetIntent(intent.getExtras());
521 }
522 } else if (Intent.ACTION_VIEW.equals(action)) {
523 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
524 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
525 xmppConnectionService
526 .getJingleConnectionManager()
527 .getTerminalSessionState(with, proposedSessionId);
528 if (terminatedRtpSession != null) {
529 // termination (due to message error or 'busy' was faster than opening the activity
530 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
531 return;
532 }
533 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
534 final RtpEndUserState state =
535 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
536 if (state != null) {
537 Log.d(Config.LOGTAG, "restored last state from intent extra");
538 updateButtonConfiguration(state);
539 updateVerifiedShield(false);
540 updateStateDisplay(state);
541 updateIncomingCallScreen(state);
542 invalidateOptionsMenu();
543 }
544 setWith(account.getRoster().getContact(with), state);
545 if (xmppConnectionService
546 .getJingleConnectionManager()
547 .fireJingleRtpConnectionStateUpdates()) {
548 return;
549 }
550 if (END_CARD.contains(state)) {
551 return;
552 }
553 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
554 final Set<Media> media = actionToMedia(lastAction);
555 if (xmppConnectionService
556 .getJingleConnectionManager()
557 .hasMatchingProposal(account, with)) {
558 putScreenInCallMode(media);
559 return;
560 }
561 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
562 finish();
563 }
564 }
565
566 private void setWidth(final RtpEndUserState state) {
567 setWith(getWith(), state);
568 }
569
570 private void setWith(final Contact contact, final RtpEndUserState state) {
571 binding.with.setText(contact.getDisplayName());
572 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
573 .contains(state)) {
574 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
575 binding.withJid.setVisibility(View.VISIBLE);
576 } else {
577 binding.withJid.setVisibility(View.GONE);
578 }
579 }
580
581 @Override
582 public void onRequestPermissionsResult(
583 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
584 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
585 final PermissionUtils.PermissionResult permissionResult =
586 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
587 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
588 if (requestCode == REQUEST_ACCEPT_CALL) {
589 acceptCall();
590 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
591 acceptContentAdd();
592 } else if (requestCode == REQUEST_ADD_CONTENT) {
593 switchToVideo();
594 }
595 } else {
596 @StringRes int res;
597 final String firstDenied =
598 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
599 if (firstDenied == null) {
600 return;
601 }
602 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
603 res = R.string.no_microphone_permission;
604 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
605 res = R.string.no_camera_permission;
606 } else {
607 throw new IllegalStateException("Invalid permission result request");
608 }
609 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
610 .show();
611 }
612 }
613
614 @Override
615 public void onStart() {
616 super.onStart();
617 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
618 mHandler.postDelayed(mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
619 this.binding.remoteVideo.setOnAspectRatioChanged(this);
620 }
621
622 @Override
623 public void onResume() {
624 super.onResume();
625 resetVisibilityExecutorShowButtons();
626 }
627
628 @Override
629 public void onStop() {
630 mHandler.removeCallbacks(mTickExecutor);
631 mHandler.removeCallbacks(mVisibilityToggleExecutor);
632 binding.remoteVideo.release();
633 binding.remoteVideo.setOnAspectRatioChanged(null);
634 binding.localVideo.release();
635 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
636 final JingleRtpConnection jingleRtpConnection =
637 weakReference == null ? null : weakReference.get();
638 if (jingleRtpConnection != null) {
639 releaseVideoTracks(jingleRtpConnection);
640 }
641 releaseProximityWakeLock();
642 super.onStop();
643 }
644
645 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
646 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
647 if (remoteVideo.isPresent()) {
648 remoteVideo.get().removeSink(binding.remoteVideo);
649 }
650 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
651 if (localVideo.isPresent()) {
652 localVideo.get().removeSink(binding.localVideo);
653 }
654 }
655
656 @Override
657 public void onBackPressed() {
658 if (isConnected()) {
659 if (switchToPictureInPicture()) {
660 return;
661 }
662 } else {
663 endCall();
664 }
665 super.onBackPressed();
666 }
667
668 @Override
669 public void onUserLeaveHint() {
670 super.onUserLeaveHint();
671 if (switchToPictureInPicture()) {
672 return;
673 }
674 // TODO apparently this method is not getting called on Android 10 when using the task
675 // switcher
676 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
677 retractSessionProposal();
678 }
679 }
680
681 private boolean isConnected() {
682 final JingleRtpConnection connection =
683 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
684 final RtpEndUserState endUserState =
685 connection == null ? null : connection.getEndUserState();
686 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
687 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
688 }
689
690 private boolean switchToPictureInPicture() {
691 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
692 if (shouldBePictureInPicture()) {
693 startPictureInPicture();
694 return true;
695 }
696 }
697 return false;
698 }
699
700 @RequiresApi(api = Build.VERSION_CODES.O)
701 private void startPictureInPicture() {
702 try {
703 final Rational rational = this.binding.remoteVideo.getAspectRatio();
704 final Rational clippedRational = Rationals.clip(rational);
705 Log.d(
706 Config.LOGTAG,
707 "suggested rational " + rational + ". clipped to " + clippedRational);
708 enterPictureInPictureMode(
709 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
710 } catch (final IllegalStateException e) {
711 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
712 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
713 }
714 }
715
716 @Override
717 public void onAspectRatioChanged(final Rational rational) {
718 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
719 final Rational clippedRational = Rationals.clip(rational);
720 Log.d(
721 Config.LOGTAG,
722 "suggested rational after aspect ratio change "
723 + rational
724 + ". clipped to "
725 + clippedRational);
726 setPictureInPictureParams(
727 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
728 }
729 }
730
731 private boolean deviceSupportsPictureInPicture() {
732 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
733 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
734 } else {
735 return false;
736 }
737 }
738
739 private boolean shouldBePictureInPicture() {
740 try {
741 final JingleRtpConnection rtpConnection = requireRtpConnection();
742 return rtpConnection.getMedia().contains(Media.VIDEO)
743 && Arrays.asList(
744 RtpEndUserState.ACCEPTING_CALL,
745 RtpEndUserState.CONNECTING,
746 RtpEndUserState.CONNECTED)
747 .contains(rtpConnection.getEndUserState());
748 } catch (final IllegalStateException e) {
749 return false;
750 }
751 }
752
753 private boolean isInConnectedVideoCall() {
754 final JingleRtpConnection rtpConnection;
755 try {
756 rtpConnection = requireRtpConnection();
757 } catch (final IllegalStateException e) {
758 return false;
759 }
760 return rtpConnection.getMedia().contains(Media.VIDEO)
761 && rtpConnection.getEndUserState() == RtpEndUserState.CONNECTED;
762 }
763
764 private boolean initializeActivityWithRunningRtpSession(
765 final Account account, Jid with, String sessionId) {
766 final WeakReference<JingleRtpConnection> reference =
767 xmppConnectionService
768 .getJingleConnectionManager()
769 .findJingleRtpConnection(account, with, sessionId);
770 if (reference == null || reference.get() == null) {
771 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
772 xmppConnectionService
773 .getJingleConnectionManager()
774 .getTerminalSessionState(with, sessionId);
775 if (terminatedRtpSession == null) {
776 throw new IllegalStateException(
777 "failed to initialize activity with running rtp session. session not found");
778 }
779 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
780 return true;
781 }
782 this.rtpConnectionReference = reference;
783 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
784 final boolean verified = requireRtpConnection().isVerified();
785 if (currentState == RtpEndUserState.ENDED) {
786 finish();
787 return true;
788 }
789 final Set<Media> media = getMedia();
790 final ContentAddition contentAddition = getPendingContentAddition();
791 if (currentState == RtpEndUserState.INCOMING_CALL) {
792 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
793 }
794 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
795 requireRtpConnection().getState())) {
796 putScreenInCallMode();
797 }
798 setWidth(currentState);
799 updateVideoViews(currentState);
800 updateStateDisplay(currentState, media, contentAddition);
801 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
802 updateButtonConfiguration(currentState, media, contentAddition);
803 updateIncomingCallScreen(currentState);
804 invalidateOptionsMenu();
805 return false;
806 }
807
808 private void initializeWithTerminatedSessionState(
809 final Account account,
810 final Jid with,
811 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
812 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
813 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
814 finish();
815 return;
816 }
817 final RtpEndUserState state = terminatedRtpSession.state;
818 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
819 updateButtonConfiguration(state);
820 updateStateDisplay(state);
821 updateIncomingCallScreen(state);
822 updateCallDuration();
823 updateVerifiedShield(false);
824 invalidateOptionsMenu();
825 setWith(account.getRoster().getContact(with), state);
826 }
827
828 private void reInitializeActivityWithRunningRtpSession(
829 final Account account, Jid with, String sessionId) {
830 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
831 resetIntent(account, with, sessionId);
832 }
833
834 private void resetIntent(final Account account, final Jid with, final String sessionId) {
835 final Intent intent = new Intent(Intent.ACTION_VIEW);
836 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
837 intent.putExtra(EXTRA_WITH, with.toEscapedString());
838 intent.putExtra(EXTRA_SESSION_ID, sessionId);
839 setIntent(intent);
840 }
841
842 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
843 surfaceViewRenderer.setVisibility(View.VISIBLE);
844 try {
845 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
846 } catch (final IllegalStateException ignored) {
847 // SurfaceViewRenderer was already initialized
848 } catch (final RuntimeException e) {
849 if (Throwables.getRootCause(e) instanceof GLException glException) {
850 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
851 }
852 }
853 surfaceViewRenderer.setEnableHardwareScaler(true);
854 }
855
856 private void updateStateDisplay(final RtpEndUserState state) {
857 updateStateDisplay(state, Collections.emptySet(), null);
858 }
859
860 private void updateStateDisplay(
861 final RtpEndUserState state,
862 final Set<Media> media,
863 final ContentAddition contentAddition) {
864 switch (state) {
865 case INCOMING_CALL -> {
866 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
867 if (media.contains(Media.VIDEO)) {
868 setTitle(R.string.rtp_state_incoming_video_call);
869 } else {
870 setTitle(R.string.rtp_state_incoming_call);
871 }
872 }
873 case INCOMING_CONTENT_ADD -> {
874 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
875 setTitle(R.string.rtp_state_content_add_video);
876 } else {
877 setTitle(R.string.rtp_state_content_add);
878 }
879 }
880 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
881 case CONNECTED -> setTitle(R.string.rtp_state_connected);
882 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
883 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
884 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
885 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
886 case RINGING -> setTitle(R.string.rtp_state_ringing);
887 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
888 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
889 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
890 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
891 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
892 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
893 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
894 case ENDED -> throw new IllegalStateException(
895 "Activity should have called finishAndReleaseWakeLock();");
896 default -> throw new IllegalStateException(
897 String.format("State %s has not been handled in UI", state));
898 }
899 }
900
901 private void updateVerifiedShield(final boolean verified) {
902 if (isPictureInPicture()) {
903 this.binding.verified.setVisibility(View.GONE);
904 return;
905 }
906 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
907 }
908
909 private void updateIncomingCallScreen(final RtpEndUserState state) {
910 updateIncomingCallScreen(state, null);
911 }
912
913 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
914 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
915 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
916 if (show) {
917 binding.contactPhoto.setVisibility(View.VISIBLE);
918 if (contact == null) {
919 AvatarWorkerTask.loadAvatar(
920 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
921 } else {
922 AvatarWorkerTask.loadAvatar(
923 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
924 }
925 } else {
926 binding.contactPhoto.setVisibility(View.GONE);
927 }
928 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
929 binding.usingAccount.setVisibility(View.VISIBLE);
930 binding.usingAccount.setText(
931 getString(
932 R.string.using_account,
933 account.getJid().asBareJid().toEscapedString()));
934 } else {
935 binding.usingAccount.setVisibility(View.GONE);
936 binding.contactPhoto.setVisibility(View.GONE);
937 }
938 }
939
940 private Set<Media> getMedia() {
941 return requireRtpConnection().getMedia();
942 }
943
944 public ContentAddition getPendingContentAddition() {
945 return requireRtpConnection().getPendingContentAddition();
946 }
947
948 private void updateButtonConfiguration(final RtpEndUserState state) {
949 updateButtonConfiguration(state, Collections.emptySet(), null);
950 }
951
952 @SuppressLint("RestrictedApi")
953 private void updateButtonConfiguration(
954 final RtpEndUserState state,
955 final Set<Media> media,
956 final ContentAddition contentAddition) {
957 if (state == RtpEndUserState.ENDING_CALL
958 || isPictureInPicture()
959 || this.buttonsHiddenAfterTimeout) {
960 this.binding.rejectCall.setVisibility(View.INVISIBLE);
961 this.binding.endCall.setVisibility(View.INVISIBLE);
962 this.binding.acceptCall.setVisibility(View.INVISIBLE);
963 } else if (state == RtpEndUserState.INCOMING_CALL) {
964 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
965 this.binding.rejectCall.setOnClickListener(this::rejectCall);
966 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
967 this.binding.rejectCall.setVisibility(View.VISIBLE);
968 this.binding.endCall.setVisibility(View.INVISIBLE);
969 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
970 this.binding.acceptCall.setOnClickListener(this::acceptCall);
971 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
972 this.binding.acceptCall.setVisibility(View.VISIBLE);
973 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
974 this.binding.rejectCall.setContentDescription(
975 getString(R.string.reject_switch_to_video));
976 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
977 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
978 this.binding.rejectCall.setVisibility(View.VISIBLE);
979 this.binding.endCall.setVisibility(View.INVISIBLE);
980 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
981 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
982 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
983 this.binding.acceptCall.setVisibility(View.VISIBLE);
984 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
985 .contains(state)) {
986 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
987 this.binding.rejectCall.setOnClickListener(this::exit);
988 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
989 this.binding.rejectCall.setVisibility(View.VISIBLE);
990 this.binding.endCall.setVisibility(View.INVISIBLE);
991 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
992 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
993 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
994 this.binding.acceptCall.setVisibility(View.VISIBLE);
995 } else if (asList(
996 RtpEndUserState.CONNECTIVITY_ERROR,
997 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
998 RtpEndUserState.APPLICATION_ERROR,
999 RtpEndUserState.RETRACTED,
1000 RtpEndUserState.SECURITY_ERROR)
1001 .contains(state)) {
1002 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1003 this.binding.rejectCall.setOnClickListener(this::exit);
1004 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1005 this.binding.rejectCall.setVisibility(View.VISIBLE);
1006 this.binding.endCall.setVisibility(View.INVISIBLE);
1007 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1008 this.binding.acceptCall.setOnClickListener(this::retry);
1009 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1010 this.binding.acceptCall.setVisibility(View.VISIBLE);
1011 } else {
1012 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1013 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1014 this.binding.endCall.setOnClickListener(this::endCall);
1015 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1016 setVisibleAndShow(this.binding.endCall);
1017 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1018 }
1019 updateInCallButtonConfiguration(state, media);
1020 }
1021
1022 private static void setVisibleAndShow(final FloatingActionButton button) {
1023 button.show();
1024 button.setVisibility(View.VISIBLE);
1025 }
1026
1027 private boolean isPictureInPicture() {
1028 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1029 return isInPictureInPictureMode();
1030 } else {
1031 return false;
1032 }
1033 }
1034
1035 private void updateInCallButtonConfiguration() {
1036 updateInCallButtonConfiguration(
1037 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1038 }
1039
1040 @SuppressLint("RestrictedApi")
1041 private void updateInCallButtonConfiguration(
1042 final RtpEndUserState state, final Set<Media> media) {
1043 final var showButtons = !isPictureInPicture() && !buttonsHiddenAfterTimeout;
1044 if (STATES_CONSIDERED_CONNECTED.contains(state) && showButtons) {
1045 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1046 if (media.contains(Media.VIDEO)) {
1047 final JingleRtpConnection rtpConnection = requireRtpConnection();
1048 updateInCallButtonConfigurationVideo(
1049 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1050 } else {
1051 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1052 updateInCallButtonConfigurationSpeaker(
1053 callIntegration.getSelectedAudioDevice(),
1054 callIntegration.getAudioDevices().size());
1055 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1056 }
1057 if (media.contains(Media.AUDIO)) {
1058 updateInCallButtonConfigurationMicrophone(
1059 requireRtpConnection().isMicrophoneEnabled());
1060 } else {
1061 this.binding.inCallActionLeft.setVisibility(View.GONE);
1062 }
1063 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1064 && showButtons
1065 && Media.audioOnly(media)) {
1066 final CallIntegration callIntegration;
1067 try {
1068 callIntegration = requireCallIntegration();
1069 } catch (final IllegalStateException e) {
1070 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1071 return;
1072 }
1073 updateInCallButtonConfigurationSpeaker(
1074 callIntegration.getSelectedAudioDevice(),
1075 callIntegration.getAudioDevices().size());
1076 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1077 } else {
1078 this.binding.inCallActionLeft.setVisibility(View.GONE);
1079 this.binding.inCallActionRight.setVisibility(View.GONE);
1080 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1081 }
1082 }
1083
1084 @SuppressLint("RestrictedApi")
1085 private void updateInCallButtonConfigurationSpeaker(
1086 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1087 switch (selectedAudioDevice) {
1088 case EARPIECE -> {
1089 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_24dp);
1090 if (numberOfChoices >= 2) {
1091 this.binding.inCallActionRight.setContentDescription(
1092 getString(R.string.call_is_using_earpiece_tap_to_switch_to_speaker));
1093 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1094 } else {
1095 this.binding.inCallActionRight.setContentDescription(
1096 getString(R.string.call_is_using_earpiece));
1097 this.binding.inCallActionRight.setOnClickListener(null);
1098 this.binding.inCallActionRight.setClickable(false);
1099 }
1100 }
1101 case WIRED_HEADSET -> {
1102 this.binding.inCallActionRight.setContentDescription(
1103 getString(R.string.call_is_using_wired_headset));
1104 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1105 this.binding.inCallActionRight.setOnClickListener(null);
1106 this.binding.inCallActionRight.setClickable(false);
1107 }
1108 case SPEAKER_PHONE -> {
1109 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1110 if (numberOfChoices >= 2) {
1111 this.binding.inCallActionRight.setContentDescription(
1112 getString(R.string.call_is_using_speaker_tap_to_switch_to_earpiece));
1113 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1114 } else {
1115 this.binding.inCallActionRight.setContentDescription(
1116 getString(R.string.call_is_using_speaker));
1117 this.binding.inCallActionRight.setOnClickListener(null);
1118 this.binding.inCallActionRight.setClickable(false);
1119 }
1120 }
1121 case BLUETOOTH -> {
1122 this.binding.inCallActionRight.setContentDescription(
1123 getString(R.string.call_is_using_bluetooth));
1124 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_24dp);
1125 this.binding.inCallActionRight.setOnClickListener(null);
1126 this.binding.inCallActionRight.setClickable(false);
1127 }
1128 }
1129 setVisibleAndShow(this.binding.inCallActionRight);
1130 }
1131
1132 @SuppressLint("RestrictedApi")
1133 private void updateInCallButtonConfigurationVideo(
1134 final boolean videoEnabled, final boolean isCameraSwitchable) {
1135 setVisibleAndShow(this.binding.inCallActionRight);
1136 if (isCameraSwitchable) {
1137 this.binding.inCallActionFarRight.setImageResource(
1138 R.drawable.ic_flip_camera_android_24dp);
1139 setVisibleAndShow(this.binding.inCallActionFarRight);
1140 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1141 this.binding.inCallActionFarRight.setContentDescription(
1142 getString(R.string.flip_camera));
1143 } else {
1144 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1145 }
1146 if (videoEnabled) {
1147 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1148 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1149 this.binding.inCallActionRight.setContentDescription(
1150 getString(R.string.video_is_enabled_tap_to_disable));
1151 } else {
1152 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1153 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1154 this.binding.inCallActionRight.setContentDescription(
1155 getString(R.string.video_is_disabled_tap_to_enable));
1156 }
1157 }
1158
1159 private void switchCamera(final View view) {
1160 resetVisibilityToggleExecutor();
1161 Futures.addCallback(
1162 requireRtpConnection().switchCamera(),
1163 new FutureCallback<>() {
1164 @Override
1165 public void onSuccess(@Nullable Boolean isFrontCamera) {
1166 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1167 }
1168
1169 @Override
1170 public void onFailure(@NonNull final Throwable throwable) {
1171 Log.d(
1172 Config.LOGTAG,
1173 "could not switch camera",
1174 Throwables.getRootCause(throwable));
1175 Toast.makeText(
1176 RtpSessionActivity.this,
1177 R.string.could_not_switch_camera,
1178 Toast.LENGTH_LONG)
1179 .show();
1180 }
1181 },
1182 MainThreadExecutor.getInstance());
1183 }
1184
1185 private void enableVideo(final View view) {
1186 resetVisibilityToggleExecutor();
1187 try {
1188 requireRtpConnection().setVideoEnabled(true);
1189 } catch (final IllegalStateException e) {
1190 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1191 return;
1192 }
1193 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1194 }
1195
1196 private void disableVideo(final View view) {
1197 resetVisibilityToggleExecutor();
1198 final JingleRtpConnection rtpConnection = requireRtpConnection();
1199 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1200 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1201 rtpConnection.retractContentAdd();
1202 return;
1203 }
1204 try {
1205 requireRtpConnection().setVideoEnabled(false);
1206 } catch (final IllegalStateException e) {
1207 Toast.makeText(this, R.string.could_not_disable_video, Toast.LENGTH_SHORT).show();
1208 return;
1209 }
1210 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1211 }
1212
1213 @SuppressLint("RestrictedApi")
1214 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1215 if (microphoneEnabled) {
1216 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1217 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1218 } else {
1219 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1220 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1221 }
1222 setVisibleAndShow(this.binding.inCallActionLeft);
1223 }
1224
1225 private void updateCallDuration() {
1226 final JingleRtpConnection connection =
1227 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1228 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1229 this.binding.duration.setVisibility(View.GONE);
1230 return;
1231 }
1232 if (connection.zeroDuration()) {
1233 this.binding.duration.setVisibility(View.GONE);
1234 } else {
1235 this.binding.duration.setText(
1236 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1237 this.binding.duration.setVisibility(View.VISIBLE);
1238 }
1239 }
1240
1241 private void resetVisibilityToggleExecutor() {
1242 mHandler.removeCallbacks(this.mVisibilityToggleExecutor);
1243 mHandler.postDelayed(this.mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
1244 }
1245
1246 private void updateButtonInVideoCallVisibility() {
1247 if (isInConnectedVideoCall()) {
1248 if (isPictureInPicture()) {
1249 return;
1250 }
1251 Log.d(Config.LOGTAG, "hiding in-call buttons after timeout was reached");
1252 hideInCallButtons();
1253 }
1254 }
1255
1256 private void hideInCallButtons() {
1257 binding.inCallActionLeft.hide();
1258 binding.endCall.hide();
1259 binding.inCallActionRight.hide();
1260 binding.inCallActionFarRight.hide();
1261 }
1262
1263 private void showInCallButtons() {
1264 this.buttonsHiddenAfterTimeout = false;
1265 final JingleRtpConnection rtpConnection;
1266 try {
1267 rtpConnection = requireRtpConnection();
1268 } catch (final IllegalStateException e) {
1269 return;
1270 }
1271 updateButtonConfiguration(
1272 rtpConnection.getEndUserState(),
1273 rtpConnection.getMedia(),
1274 rtpConnection.getPendingContentAddition());
1275 }
1276
1277 private void resetVisibilityExecutorShowButtons() {
1278 resetVisibilityToggleExecutor();
1279 showInCallButtons();
1280 }
1281
1282 private void updateVideoViews(final RtpEndUserState state) {
1283 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1284 binding.localVideo.setVisibility(View.GONE);
1285 binding.localVideo.release();
1286 binding.remoteVideoWrapper.setVisibility(View.GONE);
1287 binding.remoteVideo.release();
1288 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1289 if (isPictureInPicture()) {
1290 binding.appBarLayout.setVisibility(View.GONE);
1291 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1292 if (Arrays.asList(
1293 RtpEndUserState.APPLICATION_ERROR,
1294 RtpEndUserState.CONNECTIVITY_ERROR,
1295 RtpEndUserState.SECURITY_ERROR)
1296 .contains(state)) {
1297 binding.pipWarning.setVisibility(View.VISIBLE);
1298 binding.pipWaiting.setVisibility(View.GONE);
1299 } else {
1300 binding.pipWarning.setVisibility(View.GONE);
1301 binding.pipWaiting.setVisibility(View.GONE);
1302 }
1303 } else {
1304 binding.appBarLayout.setVisibility(View.VISIBLE);
1305 binding.pipPlaceholder.setVisibility(View.GONE);
1306 }
1307 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1308 return;
1309 }
1310 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1311 binding.localVideo.setVisibility(View.GONE);
1312 binding.remoteVideoWrapper.setVisibility(View.GONE);
1313 binding.appBarLayout.setVisibility(View.GONE);
1314 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1315 binding.pipWarning.setVisibility(View.GONE);
1316 binding.pipWaiting.setVisibility(View.VISIBLE);
1317 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1318 return;
1319 }
1320 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1321 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1322 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1323 // paint local view over remote view
1324 binding.localVideo.setZOrderMediaOverlay(true);
1325 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1326 addSink(localVideoTrack.get(), binding.localVideo);
1327 } else {
1328 binding.localVideo.setVisibility(View.GONE);
1329 }
1330 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1331 if (remoteVideoTrack.isPresent()) {
1332 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1333 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1334 binding.remoteVideo.setScalingType(
1335 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1336 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1337 if (state == RtpEndUserState.CONNECTED) {
1338 binding.appBarLayout.setVisibility(View.GONE);
1339 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1340 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1341 } else {
1342 binding.appBarLayout.setVisibility(View.VISIBLE);
1343 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1344 binding.remoteVideoWrapper.setVisibility(View.GONE);
1345 }
1346 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1347 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1348 } else {
1349 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1350 }
1351 } else {
1352 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1353 binding.remoteVideoWrapper.setVisibility(View.GONE);
1354 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1355 }
1356 }
1357
1358 private Optional<VideoTrack> getLocalVideoTrack() {
1359 final JingleRtpConnection connection =
1360 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1361 if (connection == null) {
1362 return Optional.absent();
1363 }
1364 return connection.getLocalVideoTrack();
1365 }
1366
1367 private Optional<VideoTrack> getRemoteVideoTrack() {
1368 final JingleRtpConnection connection =
1369 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1370 if (connection == null) {
1371 return Optional.absent();
1372 }
1373 return connection.getRemoteVideoTrack();
1374 }
1375
1376 private void disableMicrophone(final View view) {
1377 setMicrophoneEnabled(false);
1378 }
1379
1380 private void enableMicrophone(final View view) {
1381 setMicrophoneEnabled(true);
1382 }
1383
1384 private void setMicrophoneEnabled(final boolean enabled) {
1385 resetVisibilityExecutorShowButtons();
1386 try {
1387 final JingleRtpConnection rtpConnection = requireRtpConnection();
1388 if (rtpConnection.setMicrophoneEnabled(enabled)) {
1389 updateInCallButtonConfiguration();
1390 }
1391 } catch (final IllegalStateException e) {
1392 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1393 }
1394 }
1395
1396 private void switchToEarpiece(final View view) {
1397 try {
1398 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1399 acquireProximityWakeLock();
1400 } catch (final IllegalStateException e) {
1401 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1402 }
1403 }
1404
1405 private void switchToSpeaker(final View view) {
1406 try {
1407 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1408 releaseProximityWakeLock();
1409 } catch (final IllegalStateException e) {
1410 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1411 }
1412 }
1413
1414 private void retry(final View view) {
1415 final Intent intent = getIntent();
1416 final Account account = extractAccount(intent);
1417 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1418 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1419 final String action = intent.getAction();
1420 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1421 this.rtpConnectionReference = null;
1422 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1423 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1424 }
1425
1426 private void exit(final View view) {
1427 finish();
1428 }
1429
1430 private void recordVoiceMail(final View view) {
1431 final Intent intent = getIntent();
1432 final Account account = extractAccount(intent);
1433 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1434 final Conversation conversation =
1435 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1436 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1437 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1438 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1439 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1440 launchIntent.putExtra(
1441 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1442 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1443 startActivity(launchIntent);
1444 finish();
1445 }
1446
1447 private Contact getWith() {
1448 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1449 final Account account = id.account;
1450 return account.getRoster().getContact(id.with);
1451 }
1452
1453 private JingleRtpConnection requireRtpConnection() {
1454 final JingleRtpConnection connection =
1455 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1456 if (connection == null) {
1457 throw new IllegalStateException("No RTP connection found");
1458 }
1459 return connection;
1460 }
1461
1462 private CallIntegration requireCallIntegration() {
1463 return requireOngoingRtpSession().getCallIntegration();
1464 }
1465
1466 private OngoingRtpSession requireOngoingRtpSession() {
1467 final JingleRtpConnection connection =
1468 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1469 if (connection != null) {
1470 return connection;
1471 }
1472 final Intent currentIntent = getIntent();
1473 final String withExtra =
1474 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1475 final var account = extractAccount(currentIntent);
1476 if (withExtra == null) {
1477 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1478 }
1479 final var matching =
1480 xmppConnectionService
1481 .getJingleConnectionManager()
1482 .matchingProposal(account, Jid.of(withExtra));
1483 if (matching.isPresent()) {
1484 return matching.get();
1485 }
1486 throw new IllegalStateException("No matching session proposal");
1487 }
1488
1489 @Override
1490 public void onJingleRtpConnectionUpdate(
1491 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1492 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1493 if (END_CARD.contains(state)) {
1494 Log.d(Config.LOGTAG, "end card reached");
1495 releaseProximityWakeLock();
1496 runOnUiThread(
1497 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1498 }
1499 if (with.isBareJid()) {
1500 // TODO check for ENDED
1501 updateRtpSessionProposalState(account, with, state);
1502 return;
1503 }
1504 if (emptyReference(this.rtpConnectionReference)) {
1505 if (END_CARD.contains(state)) {
1506 Log.d(Config.LOGTAG, "not reinitializing session");
1507 return;
1508 }
1509 // this happens when going from proposed session to actual session
1510 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1511 return;
1512 }
1513 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1514 final boolean verified = requireRtpConnection().isVerified();
1515 final Set<Media> media = getMedia();
1516 lockOrientation(media);
1517 final ContentAddition contentAddition = getPendingContentAddition();
1518 final Contact contact = getWith();
1519 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1520 if (state == RtpEndUserState.ENDED) {
1521 finish();
1522 return;
1523 }
1524 resetVisibilityToggleExecutor();
1525 runOnUiThread(
1526 () -> {
1527 updateStateDisplay(state, media, contentAddition);
1528 updateVerifiedShield(
1529 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1530 updateButtonConfiguration(state, media, contentAddition);
1531 updateVideoViews(state);
1532 updateIncomingCallScreen(state, contact);
1533 invalidateOptionsMenu();
1534 });
1535 if (END_CARD.contains(state)) {
1536 final JingleRtpConnection rtpConnection = requireRtpConnection();
1537 resetIntent(account, with, state, rtpConnection.getMedia());
1538 releaseVideoTracks(rtpConnection);
1539 this.rtpConnectionReference = null;
1540 }
1541 } else {
1542 Log.d(Config.LOGTAG, "received update for other rtp session");
1543 }
1544 }
1545
1546 @Override
1547 public void onAudioDeviceChanged(
1548 final CallIntegration.AudioDevice selectedAudioDevice,
1549 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1550 Log.d(
1551 Config.LOGTAG,
1552 "onAudioDeviceChanged in activity: selected:"
1553 + selectedAudioDevice
1554 + ", available:"
1555 + availableAudioDevices);
1556 try {
1557 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1558 final RtpEndUserState endUserState;
1559 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1560 endUserState = jingleRtpConnection.getEndUserState();
1561 } else {
1562 // for session proposals all end user states are functionally the same
1563 endUserState = RtpEndUserState.RINGING;
1564 }
1565 final Set<Media> media = ongoingRtpSession.getMedia();
1566 if (END_CARD.contains(endUserState)) {
1567 Log.d(
1568 Config.LOGTAG,
1569 "onAudioDeviceChanged() nothing to do because end card has been reached");
1570 } else {
1571 if (Media.audioOnly(media)
1572 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1573 final CallIntegration callIntegration = requireCallIntegration();
1574 updateInCallButtonConfigurationSpeaker(
1575 callIntegration.getSelectedAudioDevice(),
1576 callIntegration.getAudioDevices().size());
1577 }
1578 Log.d(
1579 Config.LOGTAG,
1580 "put proximity wake lock into proper state after device update");
1581 putProximityWakeLockInProperState(selectedAudioDevice);
1582 }
1583 } catch (final IllegalStateException e) {
1584 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1585 }
1586 }
1587
1588 private void updateRtpSessionProposalState(
1589 final Account account, final Jid with, final RtpEndUserState state) {
1590 final Intent currentIntent = getIntent();
1591 final String withExtra =
1592 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1593 if (withExtra == null) {
1594 return;
1595 }
1596 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1597 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1598 runOnUiThread(
1599 () -> {
1600 updateVerifiedShield(false);
1601 updateStateDisplay(state);
1602 updateButtonConfiguration(state, media, null);
1603 updateIncomingCallScreen(state);
1604 invalidateOptionsMenu();
1605 });
1606 resetIntent(account, with, state, media);
1607 }
1608 }
1609
1610 private void resetIntent(final Bundle extras) {
1611 final Intent intent = new Intent(Intent.ACTION_VIEW);
1612 intent.putExtras(extras);
1613 setIntent(intent);
1614 }
1615
1616 private void resetIntent(
1617 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1618 final Intent intent = new Intent(Intent.ACTION_VIEW);
1619 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1620 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1621 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1622 } else {
1623 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1624 }
1625 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1626 intent.putExtra(
1627 EXTRA_LAST_ACTION,
1628 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1629 setIntent(intent);
1630 }
1631
1632 private static boolean emptyReference(final WeakReference<?> weakReference) {
1633 return weakReference == null || weakReference.get() == null;
1634 }
1635
1636 private enum Event {
1637 ON_BACKEND_CONNECTED,
1638 ON_NEW_INTENT
1639 }
1640}