1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4import static java.util.Arrays.asList;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.PictureInPictureParams;
9import android.content.ActivityNotFoundException;
10import android.content.Context;
11import android.content.Intent;
12import android.content.pm.ActivityInfo;
13import android.content.pm.PackageManager;
14import android.opengl.GLException;
15import android.os.Build;
16import android.os.Bundle;
17import android.os.Handler;
18import android.os.PowerManager;
19import android.util.Log;
20import android.util.Rational;
21import android.view.KeyEvent;
22import android.view.Menu;
23import android.view.MenuItem;
24import android.view.View;
25import android.view.WindowManager;
26import android.widget.Toast;
27import androidx.annotation.NonNull;
28import androidx.annotation.Nullable;
29import androidx.annotation.RequiresApi;
30import androidx.annotation.StringRes;
31import androidx.databinding.DataBindingUtil;
32import com.google.android.material.floatingactionbutton.FloatingActionButton;
33import com.google.common.base.Optional;
34import com.google.common.base.Preconditions;
35import com.google.common.base.Strings;
36import com.google.common.base.Throwables;
37import com.google.common.collect.ImmutableList;
38import com.google.common.collect.ImmutableSet;
39import com.google.common.util.concurrent.FutureCallback;
40import com.google.common.util.concurrent.Futures;
41import eu.siacs.conversations.Config;
42import eu.siacs.conversations.R;
43import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
44import eu.siacs.conversations.entities.Account;
45import eu.siacs.conversations.entities.Contact;
46import eu.siacs.conversations.entities.Conversation;
47import eu.siacs.conversations.services.CallIntegration;
48import eu.siacs.conversations.services.CallIntegrationConnectionService;
49import eu.siacs.conversations.services.XmppConnectionService;
50import eu.siacs.conversations.ui.util.AvatarWorkerTask;
51import eu.siacs.conversations.ui.util.MainThreadExecutor;
52import eu.siacs.conversations.ui.util.Rationals;
53import eu.siacs.conversations.utils.PermissionUtils;
54import eu.siacs.conversations.utils.TimeFrameUtils;
55import eu.siacs.conversations.xmpp.Jid;
56import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
57import eu.siacs.conversations.xmpp.jingle.ContentAddition;
58import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
59import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
60import eu.siacs.conversations.xmpp.jingle.Media;
61import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
62import eu.siacs.conversations.xmpp.jingle.RtpCapability;
63import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
64import java.lang.ref.WeakReference;
65import java.util.Arrays;
66import java.util.Collections;
67import java.util.List;
68import java.util.Set;
69import org.webrtc.RendererCommon;
70import org.webrtc.SurfaceViewRenderer;
71import org.webrtc.VideoTrack;
72
73public class RtpSessionActivity extends XmppActivity
74 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
75 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
76
77 public static final String EXTRA_WITH = "with";
78 public static final String EXTRA_SESSION_ID = "session_id";
79 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
80 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
81 public static final String EXTRA_LAST_ACTION = "last_action";
82 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
83 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
84 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
85
86 private static final int CALL_DURATION_UPDATE_INTERVAL = 250;
87 private static final int BUTTON_VISIBILITY_TIMEOUT = 10_000;
88
89 private static final List<RtpEndUserState> END_CARD =
90 Arrays.asList(
91 RtpEndUserState.APPLICATION_ERROR,
92 RtpEndUserState.SECURITY_ERROR,
93 RtpEndUserState.DECLINED_OR_BUSY,
94 RtpEndUserState.CONTACT_OFFLINE,
95 RtpEndUserState.CONNECTIVITY_ERROR,
96 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
97 RtpEndUserState.RETRACTED);
98 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
99 Arrays.asList(
100 RtpEndUserState.APPLICATION_ERROR,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.SECURITY_ERROR);
103 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
104 Arrays.asList(
105 RtpEndUserState.CONNECTING,
106 RtpEndUserState.CONNECTED,
107 RtpEndUserState.RECONNECTING,
108 RtpEndUserState.INCOMING_CONTENT_ADD);
109 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
110 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
111 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
112 Arrays.asList(
113 RtpEndUserState.ACCEPTING_CALL,
114 RtpEndUserState.CONNECTING,
115 RtpEndUserState.RECONNECTING);
116 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
117 new ImmutableList.Builder<RtpEndUserState>()
118 .add(RtpEndUserState.FINDING_DEVICE)
119 .add(RtpEndUserState.RINGING)
120 .add(RtpEndUserState.ACCEPTING_CALL)
121 .add(RtpEndUserState.CONNECTING)
122 .addAll(STATES_CONSIDERED_CONNECTED)
123 .build();
124 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
125 private static final int REQUEST_ACCEPT_CALL = 0x1111;
126 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
127 private static final int REQUEST_ADD_CONTENT = 0x1113;
128 private WeakReference<JingleRtpConnection> rtpConnectionReference;
129
130 private ActivityRtpSessionBinding binding;
131 private PowerManager.WakeLock mProximityWakeLock;
132
133 private final Handler mHandler = new Handler();
134 private final Runnable mTickExecutor =
135 new Runnable() {
136 @Override
137 public void run() {
138 updateCallDuration();
139 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
140 }
141 };
142 private boolean buttonsHiddenAfterTimeout = false;
143 private final Runnable mVisibilityToggleExecutor = this::updateButtonInVideoCallVisibility;
144
145 public static Set<Media> actionToMedia(final String action) {
146 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
147 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
148 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
149 return ImmutableSet.of(Media.AUDIO);
150 } else {
151 Log.w(
152 Config.LOGTAG,
153 "actionToMedia can not get media set from unknown action " + action);
154 return Collections.emptySet();
155 }
156 }
157
158 private static void addSink(
159 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
160 try {
161 videoTrack.addSink(surfaceViewRenderer);
162 } catch (final IllegalStateException e) {
163 Log.e(
164 Config.LOGTAG,
165 "possible race condition on trying to display video track. ignoring",
166 e);
167 }
168 }
169
170 @Override
171 public void onCreate(Bundle savedInstanceState) {
172 super.onCreate(savedInstanceState);
173 getWindow()
174 .addFlags(
175 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
176 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
177 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
178 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
179 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
180 this.binding.remoteVideo.setOnClickListener(this::onVideoScreenClick);
181 this.binding.localVideo.setOnClickListener(this::onVideoScreenClick);
182 setSupportActionBar(binding.toolbar);
183 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
184 }
185
186 private void onVideoScreenClick(final View view) {
187 resetVisibilityExecutorShowButtons();
188 }
189
190 @Override
191 public boolean onCreateOptionsMenu(final Menu menu) {
192 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
193 final MenuItem help = menu.findItem(R.id.action_help);
194 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
195 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
196 help.setVisible(Config.HELP != null && isHelpButtonVisible());
197 gotoChat.setVisible(isSwitchToConversationVisible());
198 switchToVideo.setVisible(isSwitchToVideoVisible());
199 return super.onCreateOptionsMenu(menu);
200 }
201
202 @Override
203 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
204 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
205 if (xmppConnectionService != null) {
206 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
207 return true;
208 }
209 }
210 }
211 return super.onKeyDown(keyCode, event);
212 }
213
214 private boolean isHelpButtonVisible() {
215 try {
216 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
217 } catch (IllegalStateException e) {
218 final Intent intent = getIntent();
219 final String state =
220 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
221 if (state != null) {
222 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
223 } else {
224 return false;
225 }
226 }
227 }
228
229 private boolean isSwitchToConversationVisible() {
230 final JingleRtpConnection connection =
231 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
232 return connection != null
233 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
234 }
235
236 private boolean isSwitchToVideoVisible() {
237 final JingleRtpConnection connection =
238 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
239 if (connection == null) {
240 return false;
241 }
242 return connection.isSwitchToVideoAvailable();
243 }
244
245 private void switchToConversation() {
246 final Contact contact = getWith();
247 final Conversation conversation =
248 xmppConnectionService.findOrCreateConversation(
249 contact.getAccount(), contact.getJid(), false, true);
250 switchToConversation(conversation);
251 }
252
253 public boolean onOptionsItemSelected(final MenuItem item) {
254 final var itemItem = item.getItemId();
255 if (itemItem == R.id.action_help) {
256 launchHelpInBrowser();
257 return true;
258 } else if (itemItem == R.id.action_goto_chat) {
259 switchToConversation();
260 return true;
261 } else if (itemItem == R.id.action_switch_to_video) {
262 requestPermissionAndSwitchToVideo();
263 return true;
264 } else {
265 return super.onOptionsItemSelected(item);
266 }
267 }
268
269 private void launchHelpInBrowser() {
270 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
271 try {
272 startActivity(intent);
273 } catch (final ActivityNotFoundException e) {
274 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
275 .show();
276 }
277 }
278
279 private void endCall(View view) {
280 endCall();
281 }
282
283 private void endCall() {
284 if (this.rtpConnectionReference == null) {
285 retractSessionProposal();
286 finish();
287 } else {
288 requireRtpConnection().endCall();
289 }
290 }
291
292 private void retractSessionProposal() {
293 final Intent intent = getIntent();
294 final String action = intent.getAction();
295 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
296 final Account account = extractAccount(intent);
297 final Jid with = Jid.of(intent.getStringExtra(EXTRA_WITH));
298 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
299 if (!Intent.ACTION_VIEW.equals(action)
300 || state == null
301 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
302 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
303 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
304 }
305 xmppConnectionService
306 .getJingleConnectionManager()
307 .retractSessionProposal(account, with.asBareJid());
308 }
309
310 private void rejectCall(View view) {
311 requireRtpConnection().rejectCall();
312 finish();
313 }
314
315 private void acceptCall(View view) {
316 requestPermissionsAndAcceptCall();
317 }
318
319 private void acceptContentAdd() {
320 try {
321 final ContentAddition pendingContentAddition =
322 requireRtpConnection().getPendingContentAddition();
323 if (pendingContentAddition == null) {
324 Log.d(Config.LOGTAG, "content offer was gone after granting permission");
325 return;
326 }
327 requireRtpConnection().acceptContentAdd(pendingContentAddition.summary);
328 } catch (final IllegalStateException e) {
329 Toast.makeText(this, Strings.nullToEmpty(e.getMessage()), Toast.LENGTH_SHORT).show();
330 }
331 }
332
333 private void requestPermissionAndSwitchToVideo() {
334 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
335 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
336 switchToVideo();
337 }
338 }
339
340 private void switchToVideo() {
341 try {
342 requireRtpConnection().addMedia(Media.VIDEO);
343 } catch (final IllegalStateException e) {
344 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
345 }
346 }
347
348 private void acceptContentAdd(final ContentAddition contentAddition) {
349 if (contentAddition == null
350 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
351 Log.d(Config.LOGTAG, "ignore press on content-accept button");
352 return;
353 }
354 requestPermissionAndAcceptContentAdd(contentAddition);
355 }
356
357 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
358 final List<String> permissions = permissions(contentAddition.media());
359 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
360 try {
361 requireRtpConnection().acceptContentAdd(contentAddition.summary);
362 } catch (final IllegalStateException e) {
363 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
364 }
365 }
366 }
367
368 private void rejectContentAdd(final View view) {
369 requireRtpConnection().rejectContentAdd();
370 }
371
372 private void requestPermissionsAndAcceptCall() {
373 final List<String> permissions = permissions(getMedia());
374 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
375 putScreenInCallMode();
376 acceptCall();
377 }
378 }
379
380 private List<String> permissions(final Set<Media> media) {
381 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
382 if (media.contains(Media.VIDEO)) {
383 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
384 } else {
385 permissions.add(Manifest.permission.RECORD_AUDIO);
386 }
387 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
388 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
389 }
390 return permissions.build();
391 }
392
393 private void acceptCall() {
394 try {
395 requireRtpConnection().acceptCall();
396 } catch (final IllegalStateException e) {
397 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
398 }
399 }
400
401 private void putScreenInCallMode() {
402 putScreenInCallMode(requireRtpConnection().getMedia());
403 }
404
405 private void putScreenInCallMode(final Set<Media> media) {
406 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
407 if (Media.audioOnly(media)) {
408 final JingleRtpConnection rtpConnection =
409 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
410 final CallIntegration callIntegration =
411 rtpConnection == null ? null : rtpConnection.getCallIntegration();
412 if (callIntegration == null
413 || callIntegration.getSelectedAudioDevice()
414 == CallIntegration.AudioDevice.EARPIECE) {
415 acquireProximityWakeLock();
416 }
417 }
418 lockOrientation(media);
419 }
420
421 private void lockOrientation(final Set<Media> media) {
422 if (Media.audioOnly(media)) {
423 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
424 } else {
425 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
426 }
427 }
428
429 @SuppressLint("WakelockTimeout")
430 private void acquireProximityWakeLock() {
431 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
432 if (powerManager == null) {
433 Log.e(Config.LOGTAG, "power manager not available");
434 return;
435 }
436 if (isFinishing()) {
437 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
438 return;
439 }
440 if (this.mProximityWakeLock == null) {
441 this.mProximityWakeLock =
442 powerManager.newWakeLock(
443 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
444 }
445 if (!this.mProximityWakeLock.isHeld()) {
446 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
447 this.mProximityWakeLock.acquire();
448 }
449 }
450
451 private void releaseProximityWakeLock() {
452 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
453 Log.d(Config.LOGTAG, "releasing proximity wake lock");
454 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
455 this.mProximityWakeLock = null;
456 }
457 }
458
459 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
460 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
461 acquireProximityWakeLock();
462 } else {
463 releaseProximityWakeLock();
464 }
465 }
466
467 @Override
468 protected void refreshUiReal() {}
469
470 @Override
471 public void onNewIntent(final Intent intent) {
472 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
473 super.onNewIntent(intent);
474 if (intent == null) {
475 return;
476 }
477 setIntent(intent);
478 if (xmppConnectionService == null) {
479 Log.d(
480 Config.LOGTAG,
481 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
482 return;
483 }
484 initializeWithIntent(Event.ON_NEW_INTENT, intent);
485 }
486
487 @Override
488 protected void onBackendConnected() {
489 final var intent = getIntent();
490 if (intent == null) {
491 return;
492 }
493 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
494 }
495
496 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
497 final String action = intent.getAction();
498 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
499 final Account account = extractAccount(intent);
500 final var extraWith = intent.getStringExtra(EXTRA_WITH);
501 final Jid with = Strings.isNullOrEmpty(extraWith) ? null : Jid.of(extraWith);
502 if (with == null || account == null) {
503 Log.e(Config.LOGTAG, "intent is missing extras (account or with)");
504 return;
505 }
506 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
507 if (sessionId != null) {
508 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
509 return;
510 }
511 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
512 Log.d(Config.LOGTAG, "intent action was accept");
513 requestPermissionsAndAcceptCall();
514 resetIntent(intent.getExtras());
515 }
516 } else if (Intent.ACTION_VIEW.equals(action)) {
517 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
518 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
519 xmppConnectionService
520 .getJingleConnectionManager()
521 .getTerminalSessionState(with, proposedSessionId);
522 if (terminatedRtpSession != null) {
523 // termination (due to message error or 'busy' was faster than opening the activity
524 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
525 return;
526 }
527 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
528 final RtpEndUserState state =
529 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
530 final var contact = account.getRoster().getContact(with);
531 if (state != null) {
532 Log.d(Config.LOGTAG, "restored last state from intent extra");
533 updateButtonConfiguration(state);
534 updateVerifiedShield(false);
535 updateStateDisplay(state);
536 updateIncomingCallScreen(state);
537 updateSupportWarning(state, contact);
538 invalidateOptionsMenu();
539 }
540 setWith(state, contact);
541 if (xmppConnectionService
542 .getJingleConnectionManager()
543 .fireJingleRtpConnectionStateUpdates()) {
544 return;
545 }
546 if (END_CARD.contains(state)) {
547 return;
548 }
549 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
550 final Set<Media> media = actionToMedia(lastAction);
551 if (xmppConnectionService
552 .getJingleConnectionManager()
553 .hasMatchingProposal(account, with)) {
554 putScreenInCallMode(media);
555 return;
556 }
557 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
558 finish();
559 }
560 }
561
562 private void setWidth(final RtpEndUserState state) {
563 setWith(state, getWith());
564 }
565
566 private void setWith(final RtpEndUserState state, final Contact contact) {
567 binding.with.setText(contact.getDisplayName());
568 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
569 .contains(state)) {
570 binding.withJid.setText(contact.getJid().asBareJid().toString());
571 binding.withJid.setVisibility(View.VISIBLE);
572 } else {
573 binding.withJid.setVisibility(View.GONE);
574 }
575 }
576
577 @Override
578 public void onRequestPermissionsResult(
579 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
580 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
581 final PermissionUtils.PermissionResult permissionResult =
582 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
583 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
584 if (requestCode == REQUEST_ACCEPT_CALL) {
585 acceptCall();
586 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
587 acceptContentAdd();
588 } else if (requestCode == REQUEST_ADD_CONTENT) {
589 switchToVideo();
590 }
591 } else {
592 @StringRes int res;
593 final String firstDenied =
594 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
595 if (firstDenied == null) {
596 return;
597 }
598 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
599 res = R.string.no_microphone_permission;
600 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
601 res = R.string.no_camera_permission;
602 } else {
603 throw new IllegalStateException("Invalid permission result request");
604 }
605 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
606 .show();
607 }
608 }
609
610 @Override
611 public void onStart() {
612 super.onStart();
613 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
614 mHandler.postDelayed(mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
615 this.binding.remoteVideo.setOnAspectRatioChanged(this);
616 }
617
618 @Override
619 public void onResume() {
620 super.onResume();
621 resetVisibilityExecutorShowButtons();
622 }
623
624 @Override
625 public void onStop() {
626 mHandler.removeCallbacks(mTickExecutor);
627 mHandler.removeCallbacks(mVisibilityToggleExecutor);
628 binding.remoteVideo.release();
629 binding.remoteVideo.setOnAspectRatioChanged(null);
630 binding.localVideo.release();
631 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
632 final JingleRtpConnection jingleRtpConnection =
633 weakReference == null ? null : weakReference.get();
634 if (jingleRtpConnection != null) {
635 releaseVideoTracks(jingleRtpConnection);
636 }
637 releaseProximityWakeLock();
638 super.onStop();
639 }
640
641 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
642 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
643 if (remoteVideo.isPresent()) {
644 remoteVideo.get().removeSink(binding.remoteVideo);
645 }
646 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
647 if (localVideo.isPresent()) {
648 localVideo.get().removeSink(binding.localVideo);
649 }
650 }
651
652 @Override
653 public void onBackPressed() {
654 if (isConnected()) {
655 if (switchToPictureInPicture()) {
656 return;
657 }
658 } else {
659 endCall();
660 }
661 super.onBackPressed();
662 }
663
664 @Override
665 public void onUserLeaveHint() {
666 super.onUserLeaveHint();
667 if (switchToPictureInPicture()) {
668 return;
669 }
670 // TODO apparently this method is not getting called on Android 10 when using the task
671 // switcher
672 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
673 retractSessionProposal();
674 }
675 }
676
677 private boolean isConnected() {
678 final JingleRtpConnection connection =
679 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
680 final RtpEndUserState endUserState =
681 connection == null ? null : connection.getEndUserState();
682 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
683 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
684 }
685
686 private boolean switchToPictureInPicture() {
687 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
688 if (shouldBePictureInPicture()) {
689 startPictureInPicture();
690 return true;
691 }
692 }
693 return false;
694 }
695
696 @RequiresApi(api = Build.VERSION_CODES.O)
697 private void startPictureInPicture() {
698 try {
699 final Rational rational = this.binding.remoteVideo.getAspectRatio();
700 final Rational clippedRational = Rationals.clip(rational);
701 Log.d(
702 Config.LOGTAG,
703 "suggested rational " + rational + ". clipped to " + clippedRational);
704 enterPictureInPictureMode(
705 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
706 } catch (final IllegalStateException e) {
707 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
708 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
709 }
710 }
711
712 @Override
713 public void onAspectRatioChanged(final Rational rational) {
714 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
715 final Rational clippedRational = Rationals.clip(rational);
716 Log.d(
717 Config.LOGTAG,
718 "suggested rational after aspect ratio change "
719 + rational
720 + ". clipped to "
721 + clippedRational);
722 setPictureInPictureParams(
723 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
724 }
725 }
726
727 private boolean deviceSupportsPictureInPicture() {
728 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
729 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
730 } else {
731 return false;
732 }
733 }
734
735 private boolean shouldBePictureInPicture() {
736 try {
737 final JingleRtpConnection rtpConnection = requireRtpConnection();
738 return rtpConnection.getMedia().contains(Media.VIDEO)
739 && Arrays.asList(
740 RtpEndUserState.ACCEPTING_CALL,
741 RtpEndUserState.CONNECTING,
742 RtpEndUserState.CONNECTED)
743 .contains(rtpConnection.getEndUserState());
744 } catch (final IllegalStateException e) {
745 return false;
746 }
747 }
748
749 private boolean isInConnectedVideoCall() {
750 final JingleRtpConnection rtpConnection;
751 try {
752 rtpConnection = requireRtpConnection();
753 } catch (final IllegalStateException e) {
754 return false;
755 }
756 return rtpConnection.getMedia().contains(Media.VIDEO)
757 && rtpConnection.getEndUserState() == RtpEndUserState.CONNECTED;
758 }
759
760 private boolean initializeActivityWithRunningRtpSession(
761 final Account account, Jid with, String sessionId) {
762 final WeakReference<JingleRtpConnection> reference =
763 xmppConnectionService
764 .getJingleConnectionManager()
765 .findJingleRtpConnection(account, with, sessionId);
766 if (reference == null || reference.get() == null) {
767 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
768 xmppConnectionService
769 .getJingleConnectionManager()
770 .getTerminalSessionState(with, sessionId);
771 if (terminatedRtpSession == null) {
772 throw new IllegalStateException(
773 "failed to initialize activity with running rtp session. session not"
774 + " found");
775 }
776 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
777 return true;
778 }
779 this.rtpConnectionReference = reference;
780 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
781 final boolean verified = requireRtpConnection().isVerified();
782 if (currentState == RtpEndUserState.ENDED) {
783 finish();
784 return true;
785 }
786 final Set<Media> media = getMedia();
787 final ContentAddition contentAddition = getPendingContentAddition();
788 if (currentState == RtpEndUserState.INCOMING_CALL) {
789 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
790 }
791 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
792 requireRtpConnection().getState())) {
793 putScreenInCallMode();
794 }
795 setWidth(currentState);
796 updateVideoViews(currentState);
797 updateStateDisplay(currentState, media, contentAddition);
798 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
799 updateButtonConfiguration(currentState, media, contentAddition);
800 updateIncomingCallScreen(currentState);
801 invalidateOptionsMenu();
802 return false;
803 }
804
805 private void initializeWithTerminatedSessionState(
806 final Account account,
807 final Jid with,
808 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
809 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
810 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
811 finish();
812 return;
813 }
814 final RtpEndUserState state = terminatedRtpSession.state;
815 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
816 updateButtonConfiguration(state);
817 updateStateDisplay(state);
818 updateIncomingCallScreen(state);
819 updateCallDuration();
820 updateVerifiedShield(false);
821 invalidateOptionsMenu();
822 final var contact = account.getRoster().getContact(with);
823 setWith(state, contact);
824 updateSupportWarning(state, contact);
825 }
826
827 private void reInitializeActivityWithRunningRtpSession(
828 final Account account, Jid with, String sessionId) {
829 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
830 resetIntent(account, with, sessionId);
831 }
832
833 private void resetIntent(final Account account, final Jid with, final String sessionId) {
834 final Intent intent = new Intent(Intent.ACTION_VIEW);
835 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toString());
836 intent.putExtra(EXTRA_WITH, with.toString());
837 intent.putExtra(EXTRA_SESSION_ID, sessionId);
838 setIntent(intent);
839 }
840
841 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
842 surfaceViewRenderer.setVisibility(View.VISIBLE);
843 try {
844 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
845 } catch (final IllegalStateException ignored) {
846 // SurfaceViewRenderer was already initialized
847 } catch (final RuntimeException e) {
848 if (Throwables.getRootCause(e) instanceof GLException glException) {
849 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
850 }
851 }
852 surfaceViewRenderer.setEnableHardwareScaler(true);
853 }
854
855 private void updateStateDisplay(final RtpEndUserState state) {
856 updateStateDisplay(state, Collections.emptySet(), null);
857 }
858
859 private void updateStateDisplay(
860 final RtpEndUserState state,
861 final Set<Media> media,
862 final ContentAddition contentAddition) {
863 switch (state) {
864 case INCOMING_CALL -> {
865 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
866 if (media.contains(Media.VIDEO)) {
867 setTitle(R.string.rtp_state_incoming_video_call);
868 } else {
869 setTitle(R.string.rtp_state_incoming_call);
870 }
871 }
872 case INCOMING_CONTENT_ADD -> {
873 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
874 setTitle(R.string.rtp_state_content_add_video);
875 } else {
876 setTitle(R.string.rtp_state_content_add);
877 }
878 }
879 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
880 case CONNECTED -> setTitle(R.string.rtp_state_connected);
881 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
882 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
883 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
884 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
885 case RINGING -> setTitle(R.string.rtp_state_ringing);
886 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
887 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
888 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
889 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
890 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
891 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
892 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
893 case ENDED ->
894 throw new IllegalStateException(
895 "Activity should have called finishAndReleaseWakeLock();");
896 default ->
897 throw new IllegalStateException(
898 String.format("State %s has not been handled in UI", state));
899 }
900 }
901
902 private void updateVerifiedShield(final boolean verified) {
903 if (isPictureInPicture()) {
904 this.binding.verified.setVisibility(View.GONE);
905 return;
906 }
907 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
908 }
909
910 private void updateIncomingCallScreen(final RtpEndUserState state) {
911 updateIncomingCallScreen(state, null);
912 }
913
914 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
915 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
916 final boolean show = getResources().getBoolean(R.bool.is_portrait_mode);
917 if (show) {
918 binding.contactPhoto.setVisibility(View.VISIBLE);
919 if (contact == null) {
920 AvatarWorkerTask.loadAvatar(
921 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
922 } else {
923 AvatarWorkerTask.loadAvatar(
924 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
925 }
926 } else {
927 binding.contactPhoto.setVisibility(View.GONE);
928 }
929 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
930 binding.usingAccount.setVisibility(View.VISIBLE);
931 binding.usingAccount.setText(
932 getString(R.string.using_account, account.getJid().asBareJid().toString()));
933 } else {
934 binding.usingAccount.setVisibility(View.GONE);
935 binding.contactPhoto.setVisibility(View.GONE);
936 }
937 }
938
939 private void updateSupportWarning(final RtpEndUserState state, final Contact contact) {
940 if (state == RtpEndUserState.CONNECTIVITY_ERROR
941 && getResources().getBoolean(R.bool.is_portrait_mode)) {
942 binding.supportWarning.setVisibility(
943 RtpCapability.check(contact) == RtpCapability.Capability.NONE
944 ? View.VISIBLE
945 : View.GONE);
946 } else {
947 binding.supportWarning.setVisibility(View.GONE);
948 }
949 }
950
951 private Set<Media> getMedia() {
952 return requireRtpConnection().getMedia();
953 }
954
955 public ContentAddition getPendingContentAddition() {
956 return requireRtpConnection().getPendingContentAddition();
957 }
958
959 private void updateButtonConfiguration(final RtpEndUserState state) {
960 updateButtonConfiguration(state, Collections.emptySet(), null);
961 }
962
963 @SuppressLint("RestrictedApi")
964 private void updateButtonConfiguration(
965 final RtpEndUserState state,
966 final Set<Media> media,
967 final ContentAddition contentAddition) {
968 if (state == RtpEndUserState.ENDING_CALL
969 || isPictureInPicture()
970 || this.buttonsHiddenAfterTimeout) {
971 this.binding.rejectCall.setVisibility(View.INVISIBLE);
972 this.binding.endCall.setVisibility(View.INVISIBLE);
973 this.binding.acceptCall.setVisibility(View.INVISIBLE);
974 } else if (state == RtpEndUserState.INCOMING_CALL) {
975 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
976 this.binding.rejectCall.setOnClickListener(this::rejectCall);
977 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
978 this.binding.rejectCall.setVisibility(View.VISIBLE);
979 this.binding.endCall.setVisibility(View.INVISIBLE);
980 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
981 this.binding.acceptCall.setOnClickListener(this::acceptCall);
982 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
983 this.binding.acceptCall.setVisibility(View.VISIBLE);
984 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
985 this.binding.rejectCall.setContentDescription(
986 getString(R.string.reject_switch_to_video));
987 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
988 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
989 this.binding.rejectCall.setVisibility(View.VISIBLE);
990 this.binding.endCall.setVisibility(View.INVISIBLE);
991 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
992 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
993 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
994 this.binding.acceptCall.setVisibility(View.VISIBLE);
995 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
996 .contains(state)) {
997 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
998 this.binding.rejectCall.setOnClickListener(this::exit);
999 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1000 this.binding.rejectCall.setVisibility(View.VISIBLE);
1001 this.binding.endCall.setVisibility(View.INVISIBLE);
1002 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1003 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1004 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
1005 this.binding.acceptCall.setVisibility(View.VISIBLE);
1006 } else if (asList(
1007 RtpEndUserState.CONNECTIVITY_ERROR,
1008 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1009 RtpEndUserState.APPLICATION_ERROR,
1010 RtpEndUserState.RETRACTED,
1011 RtpEndUserState.SECURITY_ERROR)
1012 .contains(state)) {
1013 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1014 this.binding.rejectCall.setOnClickListener(this::exit);
1015 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1016 this.binding.rejectCall.setVisibility(View.VISIBLE);
1017 this.binding.endCall.setVisibility(View.INVISIBLE);
1018 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1019 this.binding.acceptCall.setOnClickListener(this::retry);
1020 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1021 this.binding.acceptCall.setVisibility(View.VISIBLE);
1022 } else {
1023 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1024 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1025 this.binding.endCall.setOnClickListener(this::endCall);
1026 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1027 setVisibleAndShow(this.binding.endCall);
1028 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1029 }
1030 updateInCallButtonConfiguration(state, media);
1031 }
1032
1033 private static void setVisibleAndShow(final FloatingActionButton button) {
1034 button.show();
1035 button.setVisibility(View.VISIBLE);
1036 }
1037
1038 private boolean isPictureInPicture() {
1039 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1040 return isInPictureInPictureMode();
1041 } else {
1042 return false;
1043 }
1044 }
1045
1046 private void updateInCallButtonConfiguration() {
1047 updateInCallButtonConfiguration(
1048 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1049 }
1050
1051 @SuppressLint("RestrictedApi")
1052 private void updateInCallButtonConfiguration(
1053 final RtpEndUserState state, final Set<Media> media) {
1054 final var showButtons = !isPictureInPicture() && !buttonsHiddenAfterTimeout;
1055 if (STATES_CONSIDERED_CONNECTED.contains(state) && showButtons) {
1056 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1057 if (media.contains(Media.VIDEO)) {
1058 final JingleRtpConnection rtpConnection = requireRtpConnection();
1059 updateInCallButtonConfigurationVideo(
1060 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1061 } else {
1062 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1063 updateInCallButtonConfigurationSpeaker(
1064 callIntegration.getSelectedAudioDevice(),
1065 callIntegration.getAudioDevices().size());
1066 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1067 }
1068 if (media.contains(Media.AUDIO)) {
1069 updateInCallButtonConfigurationMicrophone(
1070 requireRtpConnection().isMicrophoneEnabled());
1071 } else {
1072 this.binding.inCallActionLeft.setVisibility(View.GONE);
1073 }
1074 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1075 && showButtons
1076 && Media.audioOnly(media)) {
1077 final CallIntegration callIntegration;
1078 try {
1079 callIntegration = requireCallIntegration();
1080 } catch (final IllegalStateException e) {
1081 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1082 return;
1083 }
1084 updateInCallButtonConfigurationSpeaker(
1085 callIntegration.getSelectedAudioDevice(),
1086 callIntegration.getAudioDevices().size());
1087 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1088 } else {
1089 this.binding.inCallActionLeft.setVisibility(View.GONE);
1090 this.binding.inCallActionRight.setVisibility(View.GONE);
1091 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1092 }
1093 }
1094
1095 @SuppressLint("RestrictedApi")
1096 private void updateInCallButtonConfigurationSpeaker(
1097 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1098 switch (selectedAudioDevice) {
1099 case EARPIECE -> {
1100 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_24dp);
1101 if (numberOfChoices >= 2) {
1102 this.binding.inCallActionRight.setContentDescription(
1103 getString(R.string.call_is_using_earpiece_tap_to_switch_to_speaker));
1104 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1105 } else {
1106 this.binding.inCallActionRight.setContentDescription(
1107 getString(R.string.call_is_using_earpiece));
1108 this.binding.inCallActionRight.setOnClickListener(null);
1109 this.binding.inCallActionRight.setClickable(false);
1110 }
1111 }
1112 case WIRED_HEADSET -> {
1113 this.binding.inCallActionRight.setContentDescription(
1114 getString(R.string.call_is_using_wired_headset));
1115 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1116 this.binding.inCallActionRight.setOnClickListener(null);
1117 this.binding.inCallActionRight.setClickable(false);
1118 }
1119 case SPEAKER_PHONE -> {
1120 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1121 if (numberOfChoices >= 2) {
1122 this.binding.inCallActionRight.setContentDescription(
1123 getString(R.string.call_is_using_speaker_tap_to_switch_to_earpiece));
1124 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1125 } else {
1126 this.binding.inCallActionRight.setContentDescription(
1127 getString(R.string.call_is_using_speaker));
1128 this.binding.inCallActionRight.setOnClickListener(null);
1129 this.binding.inCallActionRight.setClickable(false);
1130 }
1131 }
1132 case BLUETOOTH -> {
1133 this.binding.inCallActionRight.setContentDescription(
1134 getString(R.string.call_is_using_bluetooth));
1135 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_24dp);
1136 this.binding.inCallActionRight.setOnClickListener(null);
1137 this.binding.inCallActionRight.setClickable(false);
1138 }
1139 }
1140 setVisibleAndShow(this.binding.inCallActionRight);
1141 }
1142
1143 @SuppressLint("RestrictedApi")
1144 private void updateInCallButtonConfigurationVideo(
1145 final boolean videoEnabled, final boolean isCameraSwitchable) {
1146 setVisibleAndShow(this.binding.inCallActionRight);
1147 if (isCameraSwitchable) {
1148 this.binding.inCallActionFarRight.setImageResource(
1149 R.drawable.ic_flip_camera_android_24dp);
1150 setVisibleAndShow(this.binding.inCallActionFarRight);
1151 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1152 this.binding.inCallActionFarRight.setContentDescription(
1153 getString(R.string.flip_camera));
1154 } else {
1155 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1156 }
1157 if (videoEnabled) {
1158 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1159 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1160 this.binding.inCallActionRight.setContentDescription(
1161 getString(R.string.video_is_enabled_tap_to_disable));
1162 } else {
1163 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1164 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1165 this.binding.inCallActionRight.setContentDescription(
1166 getString(R.string.video_is_disabled_tap_to_enable));
1167 }
1168 }
1169
1170 private void switchCamera(final View view) {
1171 resetVisibilityToggleExecutor();
1172 Futures.addCallback(
1173 requireRtpConnection().switchCamera(),
1174 new FutureCallback<>() {
1175 @Override
1176 public void onSuccess(@Nullable Boolean isFrontCamera) {
1177 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1178 }
1179
1180 @Override
1181 public void onFailure(@NonNull final Throwable throwable) {
1182 Log.d(
1183 Config.LOGTAG,
1184 "could not switch camera",
1185 Throwables.getRootCause(throwable));
1186 Toast.makeText(
1187 RtpSessionActivity.this,
1188 R.string.could_not_switch_camera,
1189 Toast.LENGTH_LONG)
1190 .show();
1191 }
1192 },
1193 MainThreadExecutor.getInstance());
1194 // TODO ^ replace with ContextCompat.getMainExecutor(getApplication())
1195 }
1196
1197 private void enableVideo(final View view) {
1198 resetVisibilityToggleExecutor();
1199 try {
1200 requireRtpConnection().setVideoEnabled(true);
1201 } catch (final IllegalStateException e) {
1202 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1203 return;
1204 }
1205 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1206 }
1207
1208 private void disableVideo(final View view) {
1209 resetVisibilityToggleExecutor();
1210 final JingleRtpConnection rtpConnection = requireRtpConnection();
1211 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1212 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1213 rtpConnection.retractContentAdd();
1214 return;
1215 }
1216 try {
1217 requireRtpConnection().setVideoEnabled(false);
1218 } catch (final IllegalStateException e) {
1219 Toast.makeText(this, R.string.could_not_disable_video, Toast.LENGTH_SHORT).show();
1220 return;
1221 }
1222 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1223 }
1224
1225 @SuppressLint("RestrictedApi")
1226 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1227 if (microphoneEnabled) {
1228 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1229 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1230 } else {
1231 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1232 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1233 }
1234 setVisibleAndShow(this.binding.inCallActionLeft);
1235 }
1236
1237 private void updateCallDuration() {
1238 final JingleRtpConnection connection =
1239 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1240 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1241 this.binding.duration.setVisibility(View.GONE);
1242 return;
1243 }
1244 if (connection.zeroDuration()) {
1245 this.binding.duration.setVisibility(View.GONE);
1246 } else {
1247 this.binding.duration.setText(
1248 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1249 this.binding.duration.setVisibility(View.VISIBLE);
1250 }
1251 }
1252
1253 private void resetVisibilityToggleExecutor() {
1254 mHandler.removeCallbacks(this.mVisibilityToggleExecutor);
1255 mHandler.postDelayed(this.mVisibilityToggleExecutor, BUTTON_VISIBILITY_TIMEOUT);
1256 }
1257
1258 private void updateButtonInVideoCallVisibility() {
1259 if (isInConnectedVideoCall()) {
1260 if (isPictureInPicture()) {
1261 return;
1262 }
1263 Log.d(Config.LOGTAG, "hiding in-call buttons after timeout was reached");
1264 hideInCallButtons();
1265 }
1266 }
1267
1268 private void hideInCallButtons() {
1269 binding.inCallActionLeft.hide();
1270 binding.endCall.hide();
1271 binding.inCallActionRight.hide();
1272 binding.inCallActionFarRight.hide();
1273 }
1274
1275 private void showInCallButtons() {
1276 this.buttonsHiddenAfterTimeout = false;
1277 final JingleRtpConnection rtpConnection;
1278 try {
1279 rtpConnection = requireRtpConnection();
1280 } catch (final IllegalStateException e) {
1281 return;
1282 }
1283 updateButtonConfiguration(
1284 rtpConnection.getEndUserState(),
1285 rtpConnection.getMedia(),
1286 rtpConnection.getPendingContentAddition());
1287 }
1288
1289 private void resetVisibilityExecutorShowButtons() {
1290 resetVisibilityToggleExecutor();
1291 showInCallButtons();
1292 }
1293
1294 private void updateVideoViews(final RtpEndUserState state) {
1295 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1296 binding.localVideo.setVisibility(View.GONE);
1297 binding.localVideo.release();
1298 binding.remoteVideoWrapper.setVisibility(View.GONE);
1299 binding.remoteVideo.release();
1300 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1301 if (isPictureInPicture()) {
1302 binding.appBarLayout.setVisibility(View.GONE);
1303 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1304 if (Arrays.asList(
1305 RtpEndUserState.APPLICATION_ERROR,
1306 RtpEndUserState.CONNECTIVITY_ERROR,
1307 RtpEndUserState.SECURITY_ERROR)
1308 .contains(state)) {
1309 binding.pipWarning.setVisibility(View.VISIBLE);
1310 binding.pipWaiting.setVisibility(View.GONE);
1311 } else {
1312 binding.pipWarning.setVisibility(View.GONE);
1313 binding.pipWaiting.setVisibility(View.GONE);
1314 }
1315 } else {
1316 binding.appBarLayout.setVisibility(View.VISIBLE);
1317 binding.pipPlaceholder.setVisibility(View.GONE);
1318 }
1319 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1320 return;
1321 }
1322 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1323 binding.localVideo.setVisibility(View.GONE);
1324 binding.remoteVideoWrapper.setVisibility(View.GONE);
1325 binding.appBarLayout.setVisibility(View.GONE);
1326 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1327 binding.pipWarning.setVisibility(View.GONE);
1328 binding.pipWaiting.setVisibility(View.VISIBLE);
1329 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1330 return;
1331 }
1332 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1333 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1334 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1335 // paint local view over remote view
1336 binding.localVideo.setZOrderMediaOverlay(true);
1337 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1338 addSink(localVideoTrack.get(), binding.localVideo);
1339 } else {
1340 binding.localVideo.setVisibility(View.GONE);
1341 }
1342 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1343 if (remoteVideoTrack.isPresent()) {
1344 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1345 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1346 binding.remoteVideo.setScalingType(
1347 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1348 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1349 if (state == RtpEndUserState.CONNECTED) {
1350 binding.appBarLayout.setVisibility(View.GONE);
1351 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1352 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1353 } else {
1354 binding.appBarLayout.setVisibility(View.VISIBLE);
1355 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1356 binding.remoteVideoWrapper.setVisibility(View.GONE);
1357 }
1358 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1359 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1360 } else {
1361 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1362 }
1363 } else {
1364 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1365 binding.remoteVideoWrapper.setVisibility(View.GONE);
1366 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1367 }
1368 }
1369
1370 private Optional<VideoTrack> getLocalVideoTrack() {
1371 final JingleRtpConnection connection =
1372 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1373 if (connection == null) {
1374 return Optional.absent();
1375 }
1376 return connection.getLocalVideoTrack();
1377 }
1378
1379 private Optional<VideoTrack> getRemoteVideoTrack() {
1380 final JingleRtpConnection connection =
1381 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1382 if (connection == null) {
1383 return Optional.absent();
1384 }
1385 return connection.getRemoteVideoTrack();
1386 }
1387
1388 private void disableMicrophone(final View view) {
1389 setMicrophoneEnabled(false);
1390 }
1391
1392 private void enableMicrophone(final View view) {
1393 setMicrophoneEnabled(true);
1394 }
1395
1396 private void setMicrophoneEnabled(final boolean enabled) {
1397 resetVisibilityExecutorShowButtons();
1398 try {
1399 final JingleRtpConnection rtpConnection = requireRtpConnection();
1400 if (rtpConnection.setMicrophoneEnabled(enabled)) {
1401 updateInCallButtonConfiguration();
1402 }
1403 } catch (final IllegalStateException e) {
1404 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1405 }
1406 }
1407
1408 private void switchToEarpiece(final View view) {
1409 try {
1410 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1411 acquireProximityWakeLock();
1412 } catch (final IllegalStateException e) {
1413 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1414 }
1415 }
1416
1417 private void switchToSpeaker(final View view) {
1418 try {
1419 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1420 releaseProximityWakeLock();
1421 } catch (final IllegalStateException e) {
1422 Toast.makeText(this, R.string.could_not_modify_call, Toast.LENGTH_SHORT).show();
1423 }
1424 }
1425
1426 private void retry(final View view) {
1427 final Intent intent = getIntent();
1428 final Account account = extractAccount(intent);
1429 final Jid with = Jid.of(intent.getStringExtra(EXTRA_WITH));
1430 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1431 final String action = intent.getAction();
1432 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1433 this.rtpConnectionReference = null;
1434 Log.d(Config.LOGTAG, "attempting retry with " + with.toString());
1435 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1436 }
1437
1438 private void exit(final View view) {
1439 finish();
1440 }
1441
1442 private void recordVoiceMail(final View view) {
1443 final Intent intent = getIntent();
1444 final Account account = extractAccount(intent);
1445 final Jid with = Jid.of(intent.getStringExtra(EXTRA_WITH));
1446 final Conversation conversation =
1447 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1448 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1449 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1450 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1451 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1452 launchIntent.putExtra(
1453 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1454 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1455 startActivity(launchIntent);
1456 finish();
1457 }
1458
1459 private Contact getWith() {
1460 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1461 final Account account = id.account;
1462 return account.getRoster().getContact(id.with);
1463 }
1464
1465 private JingleRtpConnection requireRtpConnection() {
1466 final JingleRtpConnection connection =
1467 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1468 if (connection == null) {
1469 throw new IllegalStateException("No RTP connection found");
1470 }
1471 return connection;
1472 }
1473
1474 private CallIntegration requireCallIntegration() {
1475 return requireOngoingRtpSession().getCallIntegration();
1476 }
1477
1478 private OngoingRtpSession requireOngoingRtpSession() {
1479 final JingleRtpConnection connection =
1480 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1481 if (connection != null) {
1482 return connection;
1483 }
1484 final Intent currentIntent = getIntent();
1485 final String withExtra =
1486 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1487 final var account = extractAccount(currentIntent);
1488 if (withExtra == null) {
1489 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1490 }
1491 final var matching =
1492 xmppConnectionService
1493 .getJingleConnectionManager()
1494 .matchingProposal(account, Jid.of(withExtra));
1495 if (matching.isPresent()) {
1496 return matching.get();
1497 }
1498 throw new IllegalStateException("No matching session proposal");
1499 }
1500
1501 @Override
1502 public void onJingleRtpConnectionUpdate(
1503 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1504 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1505 if (END_CARD.contains(state)) {
1506 Log.d(Config.LOGTAG, "end card reached");
1507 releaseProximityWakeLock();
1508 runOnUiThread(
1509 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1510 }
1511 if (with.isBareJid()) {
1512 // TODO check for ENDED
1513 updateRtpSessionProposalState(account, with, state);
1514 return;
1515 }
1516 if (emptyReference(this.rtpConnectionReference)) {
1517 if (END_CARD.contains(state)) {
1518 Log.d(Config.LOGTAG, "not reinitializing session");
1519 return;
1520 }
1521 // this happens when going from proposed session to actual session
1522 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1523 return;
1524 }
1525 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1526 final boolean verified = requireRtpConnection().isVerified();
1527 final Set<Media> media = getMedia();
1528 lockOrientation(media);
1529 final ContentAddition contentAddition = getPendingContentAddition();
1530 final Contact contact = getWith();
1531 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1532 if (state == RtpEndUserState.ENDED) {
1533 finish();
1534 return;
1535 }
1536 resetVisibilityToggleExecutor();
1537 runOnUiThread(
1538 () -> {
1539 updateStateDisplay(state, media, contentAddition);
1540 updateVerifiedShield(
1541 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1542 updateButtonConfiguration(state, media, contentAddition);
1543 updateVideoViews(state);
1544 updateIncomingCallScreen(state, contact);
1545 updateSupportWarning(state, contact);
1546 invalidateOptionsMenu();
1547 });
1548 if (END_CARD.contains(state)) {
1549 final JingleRtpConnection rtpConnection = requireRtpConnection();
1550 resetIntent(account, with, state, rtpConnection.getMedia());
1551 releaseVideoTracks(rtpConnection);
1552 this.rtpConnectionReference = null;
1553 }
1554 } else {
1555 Log.d(Config.LOGTAG, "received update for other rtp session");
1556 }
1557 }
1558
1559 @Override
1560 public void onAudioDeviceChanged(
1561 final CallIntegration.AudioDevice selectedAudioDevice,
1562 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1563 Log.d(
1564 Config.LOGTAG,
1565 "onAudioDeviceChanged in activity: selected:"
1566 + selectedAudioDevice
1567 + ", available:"
1568 + availableAudioDevices);
1569 try {
1570 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1571 final RtpEndUserState endUserState;
1572 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1573 endUserState = jingleRtpConnection.getEndUserState();
1574 } else {
1575 // for session proposals all end user states are functionally the same
1576 endUserState = RtpEndUserState.RINGING;
1577 }
1578 final Set<Media> media = ongoingRtpSession.getMedia();
1579 if (END_CARD.contains(endUserState)) {
1580 Log.d(
1581 Config.LOGTAG,
1582 "onAudioDeviceChanged() nothing to do because end card has been reached");
1583 } else {
1584 if (Media.audioOnly(media)
1585 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1586 final CallIntegration callIntegration = requireCallIntegration();
1587 updateInCallButtonConfigurationSpeaker(
1588 callIntegration.getSelectedAudioDevice(),
1589 callIntegration.getAudioDevices().size());
1590 }
1591 Log.d(
1592 Config.LOGTAG,
1593 "put proximity wake lock into proper state after device update");
1594 putProximityWakeLockInProperState(selectedAudioDevice);
1595 }
1596 } catch (final IllegalStateException e) {
1597 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1598 }
1599 }
1600
1601 private void updateRtpSessionProposalState(
1602 final Account account, final Jid with, final RtpEndUserState state) {
1603 final Intent currentIntent = getIntent();
1604 final String withExtra =
1605 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1606 if (withExtra == null) {
1607 return;
1608 }
1609 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1610 if (Jid.of(withExtra).asBareJid().equals(with)) {
1611 runOnUiThread(
1612 () -> {
1613 updateVerifiedShield(false);
1614 updateStateDisplay(state);
1615 updateButtonConfiguration(state, media, null);
1616 updateIncomingCallScreen(state);
1617 updateSupportWarning(state, account.getRoster().getContact(with));
1618 invalidateOptionsMenu();
1619 });
1620 resetIntent(account, with, state, media);
1621 }
1622 }
1623
1624 private void resetIntent(final Bundle extras) {
1625 final Intent intent = new Intent(Intent.ACTION_VIEW);
1626 intent.putExtras(extras);
1627 setIntent(intent);
1628 }
1629
1630 private void resetIntent(
1631 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1632 final Intent intent = new Intent(Intent.ACTION_VIEW);
1633 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toString());
1634 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1635 intent.putExtra(EXTRA_WITH, with.asBareJid().toString());
1636 } else {
1637 intent.putExtra(EXTRA_WITH, with.toString());
1638 }
1639 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1640 intent.putExtra(
1641 EXTRA_LAST_ACTION,
1642 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1643 setIntent(intent);
1644 }
1645
1646 private static boolean emptyReference(final WeakReference<?> weakReference) {
1647 return weakReference == null || weakReference.get() == null;
1648 }
1649
1650 private enum Event {
1651 ON_BACKEND_CONNECTED,
1652 ON_NEW_INTENT
1653 }
1654}