1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import eu.siacs.conversations.Config;
44import eu.siacs.conversations.R;
45import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
46import eu.siacs.conversations.entities.Account;
47import eu.siacs.conversations.entities.Contact;
48import eu.siacs.conversations.entities.Conversation;
49import eu.siacs.conversations.services.CallIntegration;
50import eu.siacs.conversations.services.CallIntegrationConnectionService;
51import eu.siacs.conversations.services.XmppConnectionService;
52import eu.siacs.conversations.ui.util.AvatarWorkerTask;
53import eu.siacs.conversations.ui.util.MainThreadExecutor;
54import eu.siacs.conversations.ui.util.Rationals;
55import eu.siacs.conversations.utils.PermissionUtils;
56import eu.siacs.conversations.utils.TimeFrameUtils;
57import eu.siacs.conversations.xmpp.Jid;
58import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
59import eu.siacs.conversations.xmpp.jingle.ContentAddition;
60import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
61import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
62import eu.siacs.conversations.xmpp.jingle.Media;
63import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
64import eu.siacs.conversations.xmpp.jingle.RtpCapability;
65import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
66
67import org.webrtc.RendererCommon;
68import org.webrtc.SurfaceViewRenderer;
69import org.webrtc.VideoTrack;
70
71import java.lang.ref.WeakReference;
72import java.util.Arrays;
73import java.util.Collections;
74import java.util.List;
75import java.util.Set;
76
77public class RtpSessionActivity extends XmppActivity
78 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
79 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
80
81 public static final String EXTRA_WITH = "with";
82 public static final String EXTRA_SESSION_ID = "session_id";
83 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
84 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
85 public static final String EXTRA_LAST_ACTION = "last_action";
86 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
87 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
88 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
89
90 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
91
92 private static final List<RtpEndUserState> END_CARD =
93 Arrays.asList(
94 RtpEndUserState.APPLICATION_ERROR,
95 RtpEndUserState.SECURITY_ERROR,
96 RtpEndUserState.DECLINED_OR_BUSY,
97 RtpEndUserState.CONTACT_OFFLINE,
98 RtpEndUserState.CONNECTIVITY_ERROR,
99 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
100 RtpEndUserState.RETRACTED);
101 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
102 Arrays.asList(
103 RtpEndUserState.APPLICATION_ERROR,
104 RtpEndUserState.CONNECTIVITY_ERROR,
105 RtpEndUserState.SECURITY_ERROR);
106 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
107 Arrays.asList(
108 RtpEndUserState.CONNECTING,
109 RtpEndUserState.CONNECTED,
110 RtpEndUserState.RECONNECTING,
111 RtpEndUserState.INCOMING_CONTENT_ADD);
112 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
113 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
114 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
115 Arrays.asList(
116 RtpEndUserState.ACCEPTING_CALL,
117 RtpEndUserState.CONNECTING,
118 RtpEndUserState.RECONNECTING);
119 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
120 new ImmutableList.Builder<RtpEndUserState>()
121 .add(RtpEndUserState.FINDING_DEVICE)
122 .add(RtpEndUserState.RINGING)
123 .add(RtpEndUserState.ACCEPTING_CALL)
124 .add(RtpEndUserState.CONNECTING)
125 .addAll(STATES_CONSIDERED_CONNECTED)
126 .build();
127 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
128 private static final int REQUEST_ACCEPT_CALL = 0x1111;
129 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
130 private static final int REQUEST_ADD_CONTENT = 0x1113;
131 private WeakReference<JingleRtpConnection> rtpConnectionReference;
132
133 private ActivityRtpSessionBinding binding;
134 private PowerManager.WakeLock mProximityWakeLock;
135
136 private final Handler mHandler = new Handler();
137 private final Runnable mTickExecutor =
138 new Runnable() {
139 @Override
140 public void run() {
141 updateCallDuration();
142 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
143 }
144 };
145
146 public static Set<Media> actionToMedia(final String action) {
147 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
148 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
149 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
150 return ImmutableSet.of(Media.AUDIO);
151 } else {
152 Log.w(
153 Config.LOGTAG,
154 "actionToMedia can not get media set from unknown action " + action);
155 return Collections.emptySet();
156 }
157 }
158
159 private static void addSink(
160 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
161 try {
162 videoTrack.addSink(surfaceViewRenderer);
163 } catch (final IllegalStateException e) {
164 Log.e(
165 Config.LOGTAG,
166 "possible race condition on trying to display video track. ignoring",
167 e);
168 }
169 }
170
171 @Override
172 public void onCreate(Bundle savedInstanceState) {
173 super.onCreate(savedInstanceState);
174 getWindow()
175 .addFlags(
176 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
177 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
178 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
179 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
180 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
181 setSupportActionBar(binding.toolbar);
182 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
183 }
184
185 @Override
186 public boolean onCreateOptionsMenu(final Menu menu) {
187 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
188 final MenuItem help = menu.findItem(R.id.action_help);
189 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
190 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
191 help.setVisible(Config.HELP != null && isHelpButtonVisible());
192 gotoChat.setVisible(isSwitchToConversationVisible());
193 switchToVideo.setVisible(isSwitchToVideoVisible());
194 return super.onCreateOptionsMenu(menu);
195 }
196
197 @Override
198 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
199 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
200 if (xmppConnectionService != null) {
201 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
202 return true;
203 }
204 }
205 }
206 return super.onKeyDown(keyCode, event);
207 }
208
209 private boolean isHelpButtonVisible() {
210 try {
211 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
212 } catch (IllegalStateException e) {
213 final Intent intent = getIntent();
214 final String state =
215 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
216 if (state != null) {
217 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
218 } else {
219 return false;
220 }
221 }
222 }
223
224 private boolean isSwitchToConversationVisible() {
225 final JingleRtpConnection connection =
226 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
227 return connection != null
228 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
229 }
230
231 private boolean isSwitchToVideoVisible() {
232 final JingleRtpConnection connection =
233 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
234 if (connection == null) {
235 return false;
236 }
237 return connection.isSwitchToVideoAvailable();
238 }
239
240 private void switchToConversation() {
241 final Contact contact = getWith();
242 final Conversation conversation =
243 xmppConnectionService.findOrCreateConversation(
244 contact.getAccount(), contact.getJid(), false, true);
245 switchToConversation(conversation);
246 }
247
248 public boolean onOptionsItemSelected(final MenuItem item) {
249 final var itemItem = item.getItemId();
250 if (itemItem == R.id.action_help) {
251 launchHelpInBrowser();
252 return true;
253 } else if (itemItem == R.id.action_goto_chat) {
254 switchToConversation();
255 return true;
256 } else if (itemItem == R.id.action_switch_to_video) {
257 requestPermissionAndSwitchToVideo();
258 return true;
259 } else {
260 return super.onOptionsItemSelected(item);
261 }
262 }
263
264 private void launchHelpInBrowser() {
265 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
266 try {
267 startActivity(intent);
268 } catch (final ActivityNotFoundException e) {
269 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
270 .show();
271 }
272 }
273
274 private void endCall(View view) {
275 endCall();
276 }
277
278 private void endCall() {
279 if (this.rtpConnectionReference == null) {
280 retractSessionProposal();
281 finish();
282 } else {
283 requireRtpConnection().endCall();
284 }
285 }
286
287 private void retractSessionProposal() {
288 final Intent intent = getIntent();
289 final String action = intent.getAction();
290 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
291 final Account account = extractAccount(intent);
292 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
293 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
294 if (!Intent.ACTION_VIEW.equals(action)
295 || state == null
296 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
297 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
298 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
299 }
300 xmppConnectionService
301 .getJingleConnectionManager()
302 .retractSessionProposal(account, with.asBareJid());
303 }
304
305 private void rejectCall(View view) {
306 requireRtpConnection().rejectCall();
307 finish();
308 }
309
310 private void acceptCall(View view) {
311 requestPermissionsAndAcceptCall();
312 }
313
314 private void acceptContentAdd() {
315 try {
316 requireRtpConnection()
317 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
318 } catch (final IllegalStateException e) {
319 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
320 }
321 }
322
323 private void requestPermissionAndSwitchToVideo() {
324 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
325 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
326 switchToVideo();
327 }
328 }
329
330 private void switchToVideo() {
331 try {
332 requireRtpConnection().addMedia(Media.VIDEO);
333 } catch (final IllegalStateException e) {
334 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
335 }
336 }
337
338 private void acceptContentAdd(final ContentAddition contentAddition) {
339 if (contentAddition == null
340 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
341 Log.d(Config.LOGTAG, "ignore press on content-accept button");
342 return;
343 }
344 requestPermissionAndAcceptContentAdd(contentAddition);
345 }
346
347 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
348 final List<String> permissions = permissions(contentAddition.media());
349 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
350 try {
351 requireRtpConnection().acceptContentAdd(contentAddition.summary);
352 } catch (final IllegalStateException e) {
353 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
354 }
355 }
356 }
357
358 private void rejectContentAdd(final View view) {
359 requireRtpConnection().rejectContentAdd();
360 }
361
362 private void requestPermissionsAndAcceptCall() {
363 final List<String> permissions = permissions(getMedia());
364 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
365 putScreenInCallMode();
366 acceptCall();
367 }
368 }
369
370 private List<String> permissions(final Set<Media> media) {
371 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
372 if (media.contains(Media.VIDEO)) {
373 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
374 } else {
375 permissions.add(Manifest.permission.RECORD_AUDIO);
376 }
377 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
378 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
379 }
380 return permissions.build();
381 }
382
383 private void acceptCall() {
384 try {
385 requireRtpConnection().acceptCall();
386 } catch (final IllegalStateException e) {
387 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
388 }
389 }
390
391 private void putScreenInCallMode() {
392 putScreenInCallMode(requireRtpConnection().getMedia());
393 }
394
395 private void putScreenInCallMode(final Set<Media> media) {
396 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
397 if (Media.audioOnly(media)) {
398 final JingleRtpConnection rtpConnection =
399 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
400 final CallIntegration callIntegration =
401 rtpConnection == null ? null : rtpConnection.getCallIntegration();
402 if (callIntegration == null
403 || callIntegration.getSelectedAudioDevice()
404 == CallIntegration.AudioDevice.EARPIECE) {
405 acquireProximityWakeLock();
406 }
407 }
408 lockOrientation(media);
409 }
410
411 private void lockOrientation(final Set<Media> media) {
412 if (Media.audioOnly(media)) {
413 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
414 } else {
415 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
416 }
417 }
418
419 @SuppressLint("WakelockTimeout")
420 private void acquireProximityWakeLock() {
421 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
422 if (powerManager == null) {
423 Log.e(Config.LOGTAG, "power manager not available");
424 return;
425 }
426 if (isFinishing()) {
427 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
428 return;
429 }
430 if (this.mProximityWakeLock == null) {
431 this.mProximityWakeLock =
432 powerManager.newWakeLock(
433 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
434 }
435 if (!this.mProximityWakeLock.isHeld()) {
436 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
437 this.mProximityWakeLock.acquire();
438 }
439 }
440
441 private void releaseProximityWakeLock() {
442 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
443 Log.d(Config.LOGTAG, "releasing proximity wake lock");
444 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
445 this.mProximityWakeLock = null;
446 }
447 }
448
449 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
450 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
451 acquireProximityWakeLock();
452 } else {
453 releaseProximityWakeLock();
454 }
455 }
456
457 @Override
458 protected void refreshUiReal() {}
459
460 @Override
461 public void onNewIntent(final Intent intent) {
462 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
463 super.onNewIntent(intent);
464 if (intent == null) {
465 return;
466 }
467 setIntent(intent);
468 if (xmppConnectionService == null) {
469 Log.d(
470 Config.LOGTAG,
471 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
472 return;
473 }
474 initializeWithIntent(Event.ON_NEW_INTENT, intent);
475 }
476
477 @Override
478 protected void onBackendConnected() {
479 final var intent = getIntent();
480 if (intent == null) {
481 return;
482 }
483 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
484 }
485
486 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
487 final String action = intent.getAction();
488 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
489 final Account account = extractAccount(intent);
490 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
491 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
492 if (sessionId != null) {
493 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
494 return;
495 }
496 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
497 Log.d(Config.LOGTAG, "intent action was accept");
498 requestPermissionsAndAcceptCall();
499 resetIntent(intent.getExtras());
500 }
501 } else if (Intent.ACTION_VIEW.equals(action)) {
502 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
503 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
504 xmppConnectionService
505 .getJingleConnectionManager()
506 .getTerminalSessionState(with, proposedSessionId);
507 if (terminatedRtpSession != null) {
508 // termination (due to message error or 'busy' was faster than opening the activity
509 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
510 return;
511 }
512 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
513 final RtpEndUserState state =
514 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
515 if (state != null) {
516 Log.d(Config.LOGTAG, "restored last state from intent extra");
517 updateButtonConfiguration(state);
518 updateVerifiedShield(false);
519 updateStateDisplay(state);
520 updateIncomingCallScreen(state);
521 invalidateOptionsMenu();
522 }
523 setWith(account.getRoster().getContact(with), state);
524 if (xmppConnectionService
525 .getJingleConnectionManager()
526 .fireJingleRtpConnectionStateUpdates()) {
527 return;
528 }
529 if (END_CARD.contains(state)) {
530 return;
531 }
532 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
533 final Set<Media> media = actionToMedia(lastAction);
534 if (xmppConnectionService
535 .getJingleConnectionManager()
536 .hasMatchingProposal(account, with)) {
537 putScreenInCallMode(media);
538 return;
539 }
540 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
541 finish();
542 }
543 }
544
545 private void setWidth(final RtpEndUserState state) {
546 setWith(getWith(), state);
547 }
548
549 private void setWith(final Contact contact, final RtpEndUserState state) {
550 binding.with.setText(contact.getDisplayName());
551 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
552 .contains(state)) {
553 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
554 binding.withJid.setVisibility(View.VISIBLE);
555 } else {
556 binding.withJid.setVisibility(View.GONE);
557 }
558 }
559
560 @Override
561 public void onRequestPermissionsResult(
562 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
563 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
564 final PermissionUtils.PermissionResult permissionResult =
565 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
566 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
567 if (requestCode == REQUEST_ACCEPT_CALL) {
568 acceptCall();
569 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
570 acceptContentAdd();
571 } else if (requestCode == REQUEST_ADD_CONTENT) {
572 switchToVideo();
573 }
574 } else {
575 @StringRes int res;
576 final String firstDenied =
577 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
578 if (firstDenied == null) {
579 return;
580 }
581 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
582 res = R.string.no_microphone_permission;
583 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
584 res = R.string.no_camera_permission;
585 } else {
586 throw new IllegalStateException("Invalid permission result request");
587 }
588 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
589 .show();
590 }
591 }
592
593 @Override
594 public void onStart() {
595 super.onStart();
596 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
597 this.binding.remoteVideo.setOnAspectRatioChanged(this);
598 }
599
600 @Override
601 public void onStop() {
602 mHandler.removeCallbacks(mTickExecutor);
603 binding.remoteVideo.release();
604 binding.remoteVideo.setOnAspectRatioChanged(null);
605 binding.localVideo.release();
606 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
607 final JingleRtpConnection jingleRtpConnection =
608 weakReference == null ? null : weakReference.get();
609 if (jingleRtpConnection != null) {
610 releaseVideoTracks(jingleRtpConnection);
611 }
612 releaseProximityWakeLock();
613 super.onStop();
614 }
615
616 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
617 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
618 if (remoteVideo.isPresent()) {
619 remoteVideo.get().removeSink(binding.remoteVideo);
620 }
621 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
622 if (localVideo.isPresent()) {
623 localVideo.get().removeSink(binding.localVideo);
624 }
625 }
626
627 @Override
628 public void onBackPressed() {
629 if (isConnected()) {
630 if (switchToPictureInPicture()) {
631 return;
632 }
633 } else {
634 endCall();
635 }
636 super.onBackPressed();
637 }
638
639 @Override
640 public void onUserLeaveHint() {
641 super.onUserLeaveHint();
642 if (switchToPictureInPicture()) {
643 return;
644 }
645 // TODO apparently this method is not getting called on Android 10 when using the task
646 // switcher
647 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
648 retractSessionProposal();
649 }
650 }
651
652 private boolean isConnected() {
653 final JingleRtpConnection connection =
654 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
655 final RtpEndUserState endUserState =
656 connection == null ? null : connection.getEndUserState();
657 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
658 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
659 }
660
661 private boolean switchToPictureInPicture() {
662 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
663 if (shouldBePictureInPicture()) {
664 startPictureInPicture();
665 return true;
666 }
667 }
668 return false;
669 }
670
671 @RequiresApi(api = Build.VERSION_CODES.O)
672 private void startPictureInPicture() {
673 try {
674 final Rational rational = this.binding.remoteVideo.getAspectRatio();
675 final Rational clippedRational = Rationals.clip(rational);
676 Log.d(
677 Config.LOGTAG,
678 "suggested rational " + rational + ". clipped to " + clippedRational);
679 enterPictureInPictureMode(
680 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
681 } catch (final IllegalStateException e) {
682 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
683 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
684 }
685 }
686
687 @Override
688 public void onAspectRatioChanged(final Rational rational) {
689 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
690 final Rational clippedRational = Rationals.clip(rational);
691 Log.d(
692 Config.LOGTAG,
693 "suggested rational after aspect ratio change "
694 + rational
695 + ". clipped to "
696 + clippedRational);
697 setPictureInPictureParams(
698 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
699 }
700 }
701
702 private boolean deviceSupportsPictureInPicture() {
703 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
704 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
705 } else {
706 return false;
707 }
708 }
709
710 private boolean shouldBePictureInPicture() {
711 try {
712 final JingleRtpConnection rtpConnection = requireRtpConnection();
713 return rtpConnection.getMedia().contains(Media.VIDEO)
714 && Arrays.asList(
715 RtpEndUserState.ACCEPTING_CALL,
716 RtpEndUserState.CONNECTING,
717 RtpEndUserState.CONNECTED)
718 .contains(rtpConnection.getEndUserState());
719 } catch (final IllegalStateException e) {
720 return false;
721 }
722 }
723
724 private boolean initializeActivityWithRunningRtpSession(
725 final Account account, Jid with, String sessionId) {
726 final WeakReference<JingleRtpConnection> reference =
727 xmppConnectionService
728 .getJingleConnectionManager()
729 .findJingleRtpConnection(account, with, sessionId);
730 if (reference == null || reference.get() == null) {
731 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
732 xmppConnectionService
733 .getJingleConnectionManager()
734 .getTerminalSessionState(with, sessionId);
735 if (terminatedRtpSession == null) {
736 throw new IllegalStateException(
737 "failed to initialize activity with running rtp session. session not found");
738 }
739 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
740 return true;
741 }
742 this.rtpConnectionReference = reference;
743 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
744 final boolean verified = requireRtpConnection().isVerified();
745 if (currentState == RtpEndUserState.ENDED) {
746 finish();
747 return true;
748 }
749 final Set<Media> media = getMedia();
750 final ContentAddition contentAddition = getPendingContentAddition();
751 if (currentState == RtpEndUserState.INCOMING_CALL) {
752 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
753 }
754 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
755 requireRtpConnection().getState())) {
756 putScreenInCallMode();
757 }
758 setWidth(currentState);
759 updateVideoViews(currentState);
760 updateStateDisplay(currentState, media, contentAddition);
761 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
762 updateButtonConfiguration(currentState, media, contentAddition);
763 updateIncomingCallScreen(currentState);
764 invalidateOptionsMenu();
765 return false;
766 }
767
768 private void initializeWithTerminatedSessionState(
769 final Account account,
770 final Jid with,
771 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
772 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
773 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
774 finish();
775 return;
776 }
777 final RtpEndUserState state = terminatedRtpSession.state;
778 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
779 updateButtonConfiguration(state);
780 updateStateDisplay(state);
781 updateIncomingCallScreen(state);
782 updateCallDuration();
783 updateVerifiedShield(false);
784 invalidateOptionsMenu();
785 setWith(account.getRoster().getContact(with), state);
786 }
787
788 private void reInitializeActivityWithRunningRtpSession(
789 final Account account, Jid with, String sessionId) {
790 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
791 resetIntent(account, with, sessionId);
792 }
793
794 private void resetIntent(final Account account, final Jid with, final String sessionId) {
795 final Intent intent = new Intent(Intent.ACTION_VIEW);
796 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
797 intent.putExtra(EXTRA_WITH, with.toEscapedString());
798 intent.putExtra(EXTRA_SESSION_ID, sessionId);
799 setIntent(intent);
800 }
801
802 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
803 surfaceViewRenderer.setVisibility(View.VISIBLE);
804 try {
805 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
806 } catch (final IllegalStateException ignored) {
807 // SurfaceViewRenderer was already initialized
808 } catch (final RuntimeException e) {
809 if (Throwables.getRootCause(e) instanceof GLException glException) {
810 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
811 }
812 }
813 surfaceViewRenderer.setEnableHardwareScaler(true);
814 }
815
816 private void updateStateDisplay(final RtpEndUserState state) {
817 updateStateDisplay(state, Collections.emptySet(), null);
818 }
819
820 private void updateStateDisplay(
821 final RtpEndUserState state,
822 final Set<Media> media,
823 final ContentAddition contentAddition) {
824 switch (state) {
825 case INCOMING_CALL -> {
826 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
827 if (media.contains(Media.VIDEO)) {
828 setTitle(R.string.rtp_state_incoming_video_call);
829 } else {
830 setTitle(R.string.rtp_state_incoming_call);
831 }
832 }
833 case INCOMING_CONTENT_ADD -> {
834 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
835 setTitle(R.string.rtp_state_content_add_video);
836 } else {
837 setTitle(R.string.rtp_state_content_add);
838 }
839 }
840 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
841 case CONNECTED -> setTitle(R.string.rtp_state_connected);
842 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
843 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
844 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
845 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
846 case RINGING -> setTitle(R.string.rtp_state_ringing);
847 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
848 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
849 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
850 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
851 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
852 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
853 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
854 case ENDED -> throw new IllegalStateException(
855 "Activity should have called finishAndReleaseWakeLock();");
856 default -> throw new IllegalStateException(
857 String.format("State %s has not been handled in UI", state));
858 }
859 }
860
861 private void updateVerifiedShield(final boolean verified) {
862 if (isPictureInPicture()) {
863 this.binding.verified.setVisibility(View.GONE);
864 return;
865 }
866 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
867 }
868
869 private void updateIncomingCallScreen(final RtpEndUserState state) {
870 updateIncomingCallScreen(state, null);
871 }
872
873 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
874 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
875 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
876 if (show) {
877 binding.contactPhoto.setVisibility(View.VISIBLE);
878 if (contact == null) {
879 AvatarWorkerTask.loadAvatar(
880 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
881 } else {
882 AvatarWorkerTask.loadAvatar(
883 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
884 }
885 } else {
886 binding.contactPhoto.setVisibility(View.GONE);
887 }
888 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
889 binding.usingAccount.setVisibility(View.VISIBLE);
890 binding.usingAccount.setText(
891 getString(
892 R.string.using_account,
893 account.getJid().asBareJid().toEscapedString()));
894 } else {
895 binding.usingAccount.setVisibility(View.GONE);
896 binding.contactPhoto.setVisibility(View.GONE);
897 }
898 }
899
900 private Set<Media> getMedia() {
901 return requireRtpConnection().getMedia();
902 }
903
904 public ContentAddition getPendingContentAddition() {
905 return requireRtpConnection().getPendingContentAddition();
906 }
907
908 private void updateButtonConfiguration(final RtpEndUserState state) {
909 updateButtonConfiguration(state, Collections.emptySet(), null);
910 }
911
912 @SuppressLint("RestrictedApi")
913 private void updateButtonConfiguration(
914 final RtpEndUserState state,
915 final Set<Media> media,
916 final ContentAddition contentAddition) {
917 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
918 this.binding.rejectCall.setVisibility(View.INVISIBLE);
919 this.binding.endCall.setVisibility(View.INVISIBLE);
920 this.binding.acceptCall.setVisibility(View.INVISIBLE);
921 } else if (state == RtpEndUserState.INCOMING_CALL) {
922 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
923 this.binding.rejectCall.setOnClickListener(this::rejectCall);
924 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
925 this.binding.rejectCall.setVisibility(View.VISIBLE);
926 this.binding.endCall.setVisibility(View.INVISIBLE);
927 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
928 this.binding.acceptCall.setOnClickListener(this::acceptCall);
929 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
930 this.binding.acceptCall.setVisibility(View.VISIBLE);
931 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
932 this.binding.rejectCall.setContentDescription(
933 getString(R.string.reject_switch_to_video));
934 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
935 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
936 this.binding.rejectCall.setVisibility(View.VISIBLE);
937 this.binding.endCall.setVisibility(View.INVISIBLE);
938 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
939 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
940 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
941 this.binding.acceptCall.setVisibility(View.VISIBLE);
942 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
943 .contains(state)) {
944 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
945 this.binding.rejectCall.setOnClickListener(this::exit);
946 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
947 this.binding.rejectCall.setVisibility(View.VISIBLE);
948 this.binding.endCall.setVisibility(View.INVISIBLE);
949 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
950 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
951 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
952 this.binding.acceptCall.setVisibility(View.VISIBLE);
953 } else if (asList(
954 RtpEndUserState.CONNECTIVITY_ERROR,
955 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
956 RtpEndUserState.APPLICATION_ERROR,
957 RtpEndUserState.RETRACTED,
958 RtpEndUserState.SECURITY_ERROR)
959 .contains(state)) {
960 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
961 this.binding.rejectCall.setOnClickListener(this::exit);
962 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
963 this.binding.rejectCall.setVisibility(View.VISIBLE);
964 this.binding.endCall.setVisibility(View.INVISIBLE);
965 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
966 this.binding.acceptCall.setOnClickListener(this::retry);
967 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
968 this.binding.acceptCall.setVisibility(View.VISIBLE);
969 } else {
970 this.binding.rejectCall.setVisibility(View.INVISIBLE);
971 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
972 this.binding.endCall.setOnClickListener(this::endCall);
973 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
974 this.binding.endCall.setVisibility(View.VISIBLE);
975 this.binding.acceptCall.setVisibility(View.INVISIBLE);
976 }
977 updateInCallButtonConfiguration(state, media);
978 }
979
980 private boolean isPictureInPicture() {
981 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
982 return isInPictureInPictureMode();
983 } else {
984 return false;
985 }
986 }
987
988 private void updateInCallButtonConfiguration() {
989 updateInCallButtonConfiguration(
990 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
991 }
992
993 @SuppressLint("RestrictedApi")
994 private void updateInCallButtonConfiguration(
995 final RtpEndUserState state, final Set<Media> media) {
996 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
997 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
998 if (media.contains(Media.VIDEO)) {
999 final JingleRtpConnection rtpConnection = requireRtpConnection();
1000 updateInCallButtonConfigurationVideo(
1001 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1002 } else {
1003 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1004 updateInCallButtonConfigurationSpeaker(
1005 callIntegration.getSelectedAudioDevice(),
1006 callIntegration.getAudioDevices().size());
1007 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1008 }
1009 if (media.contains(Media.AUDIO)) {
1010 updateInCallButtonConfigurationMicrophone(
1011 requireRtpConnection().isMicrophoneEnabled());
1012 } else {
1013 this.binding.inCallActionLeft.setVisibility(View.GONE);
1014 }
1015 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1016 && !isPictureInPicture()
1017 && Media.audioOnly(media)) {
1018 final CallIntegration callIntegration;
1019 try {
1020 callIntegration = requireCallIntegration();
1021 } catch (final IllegalStateException e) {
1022 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1023 return;
1024 }
1025 updateInCallButtonConfigurationSpeaker(
1026 callIntegration.getSelectedAudioDevice(),
1027 callIntegration.getAudioDevices().size());
1028 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1029 } else {
1030 this.binding.inCallActionLeft.setVisibility(View.GONE);
1031 this.binding.inCallActionRight.setVisibility(View.GONE);
1032 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1033 }
1034 }
1035
1036 @SuppressLint("RestrictedApi")
1037 private void updateInCallButtonConfigurationSpeaker(
1038 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1039 switch (selectedAudioDevice) {
1040 case EARPIECE -> {
1041 this.binding.inCallActionRight.setImageResource(
1042 R.drawable.ic_volume_off_24dp);
1043 if (numberOfChoices >= 2) {
1044 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1045 } else {
1046 this.binding.inCallActionRight.setOnClickListener(null);
1047 this.binding.inCallActionRight.setClickable(false);
1048 }
1049 }
1050 case WIRED_HEADSET -> {
1051 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1052 this.binding.inCallActionRight.setOnClickListener(null);
1053 this.binding.inCallActionRight.setClickable(false);
1054 }
1055 case SPEAKER_PHONE -> {
1056 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1057 if (numberOfChoices >= 2) {
1058 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1059 } else {
1060 this.binding.inCallActionRight.setOnClickListener(null);
1061 this.binding.inCallActionRight.setClickable(false);
1062 }
1063 }
1064 case BLUETOOTH -> {
1065 this.binding.inCallActionRight.setImageResource(
1066 R.drawable.ic_bluetooth_audio_24dp);
1067 this.binding.inCallActionRight.setOnClickListener(null);
1068 this.binding.inCallActionRight.setClickable(false);
1069 }
1070 }
1071 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1072 }
1073
1074 @SuppressLint("RestrictedApi")
1075 private void updateInCallButtonConfigurationVideo(
1076 final boolean videoEnabled, final boolean isCameraSwitchable) {
1077 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1078 if (isCameraSwitchable) {
1079 this.binding.inCallActionFarRight.setImageResource(
1080 R.drawable.ic_flip_camera_android_24dp);
1081 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1082 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1083 } else {
1084 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1085 }
1086 if (videoEnabled) {
1087 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1088 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1089 } else {
1090 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1091 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1092 }
1093 }
1094
1095 private void switchCamera(final View view) {
1096 Futures.addCallback(
1097 requireRtpConnection().switchCamera(),
1098 new FutureCallback<>() {
1099 @Override
1100 public void onSuccess(@Nullable Boolean isFrontCamera) {
1101 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1102 }
1103
1104 @Override
1105 public void onFailure(@NonNull final Throwable throwable) {
1106 Log.d(
1107 Config.LOGTAG,
1108 "could not switch camera",
1109 Throwables.getRootCause(throwable));
1110 Toast.makeText(
1111 RtpSessionActivity.this,
1112 R.string.could_not_switch_camera,
1113 Toast.LENGTH_LONG)
1114 .show();
1115 }
1116 },
1117 MainThreadExecutor.getInstance());
1118 }
1119
1120 private void enableVideo(View view) {
1121 try {
1122 requireRtpConnection().setVideoEnabled(true);
1123 } catch (final IllegalStateException e) {
1124 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1125 return;
1126 }
1127 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1128 }
1129
1130 private void disableVideo(View view) {
1131 final JingleRtpConnection rtpConnection = requireRtpConnection();
1132 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1133 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1134 rtpConnection.retractContentAdd();
1135 return;
1136 }
1137 requireRtpConnection().setVideoEnabled(false);
1138 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1139 }
1140
1141 @SuppressLint("RestrictedApi")
1142 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1143 if (microphoneEnabled) {
1144 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1145 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1146 } else {
1147 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1148 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1149 }
1150 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1151 }
1152
1153 private void updateCallDuration() {
1154 final JingleRtpConnection connection =
1155 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1156 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1157 this.binding.duration.setVisibility(View.GONE);
1158 return;
1159 }
1160 if (connection.zeroDuration()) {
1161 this.binding.duration.setVisibility(View.GONE);
1162 } else {
1163 this.binding.duration.setText(
1164 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1165 this.binding.duration.setVisibility(View.VISIBLE);
1166 }
1167 }
1168
1169 private void updateVideoViews(final RtpEndUserState state) {
1170 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1171 binding.localVideo.setVisibility(View.GONE);
1172 binding.localVideo.release();
1173 binding.remoteVideoWrapper.setVisibility(View.GONE);
1174 binding.remoteVideo.release();
1175 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1176 if (isPictureInPicture()) {
1177 binding.appBarLayout.setVisibility(View.GONE);
1178 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1179 if (Arrays.asList(
1180 RtpEndUserState.APPLICATION_ERROR,
1181 RtpEndUserState.CONNECTIVITY_ERROR,
1182 RtpEndUserState.SECURITY_ERROR)
1183 .contains(state)) {
1184 binding.pipWarning.setVisibility(View.VISIBLE);
1185 binding.pipWaiting.setVisibility(View.GONE);
1186 } else {
1187 binding.pipWarning.setVisibility(View.GONE);
1188 binding.pipWaiting.setVisibility(View.GONE);
1189 }
1190 } else {
1191 binding.appBarLayout.setVisibility(View.VISIBLE);
1192 binding.pipPlaceholder.setVisibility(View.GONE);
1193 }
1194 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1195 return;
1196 }
1197 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1198 binding.localVideo.setVisibility(View.GONE);
1199 binding.remoteVideoWrapper.setVisibility(View.GONE);
1200 binding.appBarLayout.setVisibility(View.GONE);
1201 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1202 binding.pipWarning.setVisibility(View.GONE);
1203 binding.pipWaiting.setVisibility(View.VISIBLE);
1204 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1205 return;
1206 }
1207 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1208 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1209 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1210 // paint local view over remote view
1211 binding.localVideo.setZOrderMediaOverlay(true);
1212 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1213 addSink(localVideoTrack.get(), binding.localVideo);
1214 } else {
1215 binding.localVideo.setVisibility(View.GONE);
1216 }
1217 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1218 if (remoteVideoTrack.isPresent()) {
1219 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1220 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1221 binding.remoteVideo.setScalingType(
1222 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1223 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1224 if (state == RtpEndUserState.CONNECTED) {
1225 binding.appBarLayout.setVisibility(View.GONE);
1226 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1227 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1228 } else {
1229 binding.appBarLayout.setVisibility(View.VISIBLE);
1230 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1231 binding.remoteVideoWrapper.setVisibility(View.GONE);
1232 }
1233 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1234 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1235 } else {
1236 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1237 }
1238 } else {
1239 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1240 binding.remoteVideoWrapper.setVisibility(View.GONE);
1241 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1242 }
1243 }
1244
1245 private Optional<VideoTrack> getLocalVideoTrack() {
1246 final JingleRtpConnection connection =
1247 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1248 if (connection == null) {
1249 return Optional.absent();
1250 }
1251 return connection.getLocalVideoTrack();
1252 }
1253
1254 private Optional<VideoTrack> getRemoteVideoTrack() {
1255 final JingleRtpConnection connection =
1256 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1257 if (connection == null) {
1258 return Optional.absent();
1259 }
1260 return connection.getRemoteVideoTrack();
1261 }
1262
1263 private void disableMicrophone(View view) {
1264 final JingleRtpConnection rtpConnection = requireRtpConnection();
1265 if (rtpConnection.setMicrophoneEnabled(false)) {
1266 updateInCallButtonConfiguration();
1267 }
1268 }
1269
1270 private void enableMicrophone(View view) {
1271 final JingleRtpConnection rtpConnection = requireRtpConnection();
1272 if (rtpConnection.setMicrophoneEnabled(true)) {
1273 updateInCallButtonConfiguration();
1274 }
1275 }
1276
1277 private void switchToEarpiece(final View view) {
1278 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1279 acquireProximityWakeLock();
1280 }
1281
1282 private void switchToSpeaker(final View view) {
1283 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1284 releaseProximityWakeLock();
1285 }
1286
1287 private void retry(final View view) {
1288 final Intent intent = getIntent();
1289 final Account account = extractAccount(intent);
1290 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1291 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1292 final String action = intent.getAction();
1293 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1294 this.rtpConnectionReference = null;
1295 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1296 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1297 }
1298
1299 private void exit(final View view) {
1300 finish();
1301 }
1302
1303 private void recordVoiceMail(final View view) {
1304 final Intent intent = getIntent();
1305 final Account account = extractAccount(intent);
1306 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1307 final Conversation conversation =
1308 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1309 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1310 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1311 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1312 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1313 launchIntent.putExtra(
1314 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1315 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1316 startActivity(launchIntent);
1317 finish();
1318 }
1319
1320 private Contact getWith() {
1321 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1322 final Account account = id.account;
1323 return account.getRoster().getContact(id.with);
1324 }
1325
1326 private JingleRtpConnection requireRtpConnection() {
1327 final JingleRtpConnection connection =
1328 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1329 if (connection == null) {
1330 throw new IllegalStateException("No RTP connection found");
1331 }
1332 return connection;
1333 }
1334
1335 private CallIntegration requireCallIntegration() {
1336 return requireOngoingRtpSession().getCallIntegration();
1337 }
1338
1339 private OngoingRtpSession requireOngoingRtpSession() {
1340 final JingleRtpConnection connection =
1341 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1342 if (connection != null) {
1343 return connection;
1344 }
1345 final Intent currentIntent = getIntent();
1346 final String withExtra =
1347 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1348 final var account = extractAccount(currentIntent);
1349 if (withExtra == null) {
1350 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1351 }
1352 final var matching =
1353 xmppConnectionService
1354 .getJingleConnectionManager()
1355 .matchingProposal(account, Jid.of(withExtra));
1356 if (matching.isPresent()) {
1357 return matching.get();
1358 }
1359 throw new IllegalStateException("No matching session proposal");
1360 }
1361
1362 @Override
1363 public void onJingleRtpConnectionUpdate(
1364 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1365 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1366 if (END_CARD.contains(state)) {
1367 Log.d(Config.LOGTAG, "end card reached");
1368 releaseProximityWakeLock();
1369 runOnUiThread(
1370 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1371 }
1372 if (with.isBareJid()) {
1373 updateRtpSessionProposalState(account, with, state);
1374 return;
1375 }
1376 if (emptyReference(this.rtpConnectionReference)) {
1377 if (END_CARD.contains(state)) {
1378 Log.d(Config.LOGTAG, "not reinitializing session");
1379 return;
1380 }
1381 // this happens when going from proposed session to actual session
1382 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1383 return;
1384 }
1385 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1386 final boolean verified = requireRtpConnection().isVerified();
1387 final Set<Media> media = getMedia();
1388 lockOrientation(media);
1389 final ContentAddition contentAddition = getPendingContentAddition();
1390 final Contact contact = getWith();
1391 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1392 if (state == RtpEndUserState.ENDED) {
1393 finish();
1394 return;
1395 }
1396 runOnUiThread(
1397 () -> {
1398 updateStateDisplay(state, media, contentAddition);
1399 updateVerifiedShield(
1400 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1401 updateButtonConfiguration(state, media, contentAddition);
1402 updateVideoViews(state);
1403 updateIncomingCallScreen(state, contact);
1404 invalidateOptionsMenu();
1405 });
1406 if (END_CARD.contains(state)) {
1407 final JingleRtpConnection rtpConnection = requireRtpConnection();
1408 resetIntent(account, with, state, rtpConnection.getMedia());
1409 releaseVideoTracks(rtpConnection);
1410 this.rtpConnectionReference = null;
1411 }
1412 } else {
1413 Log.d(Config.LOGTAG, "received update for other rtp session");
1414 }
1415 }
1416
1417 @Override
1418 public void onAudioDeviceChanged(
1419 final CallIntegration.AudioDevice selectedAudioDevice,
1420 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1421 Log.d(
1422 Config.LOGTAG,
1423 "onAudioDeviceChanged in activity: selected:"
1424 + selectedAudioDevice
1425 + ", available:"
1426 + availableAudioDevices);
1427 try {
1428 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1429 final RtpEndUserState endUserState;
1430 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1431 endUserState = jingleRtpConnection.getEndUserState();
1432 } else {
1433 // for session proposals all end user states are functionally the same
1434 endUserState = RtpEndUserState.RINGING;
1435 }
1436 final Set<Media> media = ongoingRtpSession.getMedia();
1437 if (END_CARD.contains(endUserState)) {
1438 Log.d(
1439 Config.LOGTAG,
1440 "onAudioDeviceChanged() nothing to do because end card has been reached");
1441 } else {
1442 if (Media.audioOnly(media)
1443 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1444 final CallIntegration callIntegration = requireCallIntegration();
1445 updateInCallButtonConfigurationSpeaker(
1446 callIntegration.getSelectedAudioDevice(),
1447 callIntegration.getAudioDevices().size());
1448 }
1449 Log.d(
1450 Config.LOGTAG,
1451 "put proximity wake lock into proper state after device update");
1452 putProximityWakeLockInProperState(selectedAudioDevice);
1453 }
1454 } catch (final IllegalStateException e) {
1455 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1456 }
1457 }
1458
1459 private void updateRtpSessionProposalState(
1460 final Account account, final Jid with, final RtpEndUserState state) {
1461 final Intent currentIntent = getIntent();
1462 final String withExtra =
1463 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1464 if (withExtra == null) {
1465 return;
1466 }
1467 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1468 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1469 runOnUiThread(
1470 () -> {
1471 updateVerifiedShield(false);
1472 updateStateDisplay(state);
1473 updateButtonConfiguration(state, media, null);
1474 updateIncomingCallScreen(state);
1475 invalidateOptionsMenu();
1476 });
1477 resetIntent(account, with, state, media);
1478 }
1479 }
1480
1481 private void resetIntent(final Bundle extras) {
1482 final Intent intent = new Intent(Intent.ACTION_VIEW);
1483 intent.putExtras(extras);
1484 setIntent(intent);
1485 }
1486
1487 private void resetIntent(
1488 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1489 final Intent intent = new Intent(Intent.ACTION_VIEW);
1490 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1491 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1492 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1493 } else {
1494 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1495 }
1496 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1497 intent.putExtra(
1498 EXTRA_LAST_ACTION,
1499 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1500 setIntent(intent);
1501 }
1502
1503 private static boolean emptyReference(final WeakReference<?> weakReference) {
1504 return weakReference == null || weakReference.get() == null;
1505 }
1506
1507 private enum Event {
1508 ON_BACKEND_CONNECTED,
1509 ON_NEW_INTENT
1510 }
1511}