1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import org.jetbrains.annotations.NotNull;
44import org.webrtc.RendererCommon;
45import org.webrtc.SurfaceViewRenderer;
46import org.webrtc.VideoTrack;
47
48import java.lang.ref.WeakReference;
49import java.util.Arrays;
50import java.util.Collections;
51import java.util.List;
52import java.util.Set;
53
54import eu.siacs.conversations.Config;
55import eu.siacs.conversations.R;
56import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
57import eu.siacs.conversations.entities.Account;
58import eu.siacs.conversations.entities.Contact;
59import eu.siacs.conversations.entities.Conversation;
60import eu.siacs.conversations.services.CallIntegration;
61import eu.siacs.conversations.services.CallIntegrationConnectionService;
62import eu.siacs.conversations.services.XmppConnectionService;
63import eu.siacs.conversations.ui.widget.DialpadView;
64import eu.siacs.conversations.ui.util.AvatarWorkerTask;
65import eu.siacs.conversations.ui.util.MainThreadExecutor;
66import eu.siacs.conversations.ui.util.Rationals;
67import eu.siacs.conversations.utils.PermissionUtils;
68import eu.siacs.conversations.utils.TimeFrameUtils;
69import eu.siacs.conversations.xmpp.Jid;
70import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
71import eu.siacs.conversations.xmpp.jingle.ContentAddition;
72import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
73import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
74import eu.siacs.conversations.xmpp.jingle.Media;
75import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
76import eu.siacs.conversations.xmpp.jingle.RtpCapability;
77import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
78
79import org.webrtc.RendererCommon;
80import org.webrtc.SurfaceViewRenderer;
81import org.webrtc.VideoTrack;
82
83import java.lang.ref.WeakReference;
84import java.util.Arrays;
85import java.util.Collections;
86import java.util.List;
87import java.util.Set;
88
89public class RtpSessionActivity extends XmppActivity
90 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
91 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
92
93 public static final String EXTRA_WITH = "with";
94 public static final String EXTRA_SESSION_ID = "session_id";
95 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
96 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
97 public static final String EXTRA_LAST_ACTION = "last_action";
98 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
99 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
100 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
101
102 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
103
104 public static final List<RtpEndUserState> END_CARD =
105 Arrays.asList(
106 RtpEndUserState.APPLICATION_ERROR,
107 RtpEndUserState.SECURITY_ERROR,
108 RtpEndUserState.DECLINED_OR_BUSY,
109 RtpEndUserState.CONTACT_OFFLINE,
110 RtpEndUserState.CONNECTIVITY_ERROR,
111 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
112 RtpEndUserState.RETRACTED);
113 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
114 Arrays.asList(
115 RtpEndUserState.APPLICATION_ERROR,
116 RtpEndUserState.CONNECTIVITY_ERROR,
117 RtpEndUserState.SECURITY_ERROR);
118 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
119 Arrays.asList(
120 RtpEndUserState.CONNECTING,
121 RtpEndUserState.CONNECTED,
122 RtpEndUserState.RECONNECTING,
123 RtpEndUserState.INCOMING_CONTENT_ADD);
124 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
125 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
126 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
127 Arrays.asList(
128 RtpEndUserState.ACCEPTING_CALL,
129 RtpEndUserState.CONNECTING,
130 RtpEndUserState.RECONNECTING);
131 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
132 new ImmutableList.Builder<RtpEndUserState>()
133 .add(RtpEndUserState.FINDING_DEVICE)
134 .add(RtpEndUserState.RINGING)
135 .add(RtpEndUserState.ACCEPTING_CALL)
136 .add(RtpEndUserState.CONNECTING)
137 .addAll(STATES_CONSIDERED_CONNECTED)
138 .build();
139 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
140 private static final int REQUEST_ACCEPT_CALL = 0x1111;
141 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
142 private static final int REQUEST_ADD_CONTENT = 0x1113;
143 private WeakReference<JingleRtpConnection> rtpConnectionReference;
144
145 private ActivityRtpSessionBinding binding;
146 private PowerManager.WakeLock mProximityWakeLock;
147
148 private final Handler mHandler = new Handler();
149 private final Runnable mTickExecutor =
150 new Runnable() {
151 @Override
152 public void run() {
153 updateCallDuration();
154 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
155 }
156 };
157
158 public static Set<Media> actionToMedia(final String action) {
159 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
160 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
161 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
162 return ImmutableSet.of(Media.AUDIO);
163 } else {
164 Log.w(
165 Config.LOGTAG,
166 "actionToMedia can not get media set from unknown action " + action);
167 return Collections.emptySet();
168 }
169 }
170
171 private static void addSink(
172 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
173 try {
174 videoTrack.addSink(surfaceViewRenderer);
175 } catch (final IllegalStateException e) {
176 Log.e(
177 Config.LOGTAG,
178 "possible race condition on trying to display video track. ignoring",
179 e);
180 }
181 }
182
183 @Override
184 public void onCreate(Bundle savedInstanceState) {
185 super.onCreate(savedInstanceState);
186 getWindow()
187 .addFlags(
188 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
189 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
190 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
191 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
192 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
193 setSupportActionBar(binding.toolbar);
194
195 binding.dialpad.setClickConsumer(tag -> {
196 final JingleRtpConnection connection =
197 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
198 if (connection != null) connection.applyDtmfTone(tag);
199 });
200
201 if (savedInstanceState != null) {
202 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
203 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
204 }
205
206 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
207 }
208
209 @Override
210 public boolean onCreateOptionsMenu(final Menu menu) {
211 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
212 final MenuItem help = menu.findItem(R.id.action_help);
213 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
214 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
215 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
216 help.setVisible(Config.HELP != null && isHelpButtonVisible());
217 gotoChat.setVisible(isSwitchToConversationVisible());
218 switchToVideo.setVisible(isSwitchToVideoVisible());
219 dialpad.setVisible(isAudioOnlyConversation());
220 return super.onCreateOptionsMenu(menu);
221 }
222
223 @Override
224 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
225 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
226 if (xmppConnectionService != null) {
227 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
228 return true;
229 }
230 }
231 }
232 return super.onKeyDown(keyCode, event);
233 }
234
235 private boolean isHelpButtonVisible() {
236 try {
237 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
238 } catch (IllegalStateException e) {
239 final Intent intent = getIntent();
240 final String state =
241 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
242 if (state != null) {
243 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
244 } else {
245 return false;
246 }
247 }
248 }
249
250 private boolean isSwitchToConversationVisible() {
251 final JingleRtpConnection connection =
252 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
253 return connection != null
254 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
255 }
256
257 private boolean isAudioOnlyConversation() {
258 final JingleRtpConnection connection =
259 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
260
261 return connection != null && !connection.getMedia().contains(Media.VIDEO);
262 }
263
264 private boolean isSwitchToVideoVisible() {
265 final JingleRtpConnection connection =
266 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
267 if (connection == null) {
268 return false;
269 }
270 return connection.isSwitchToVideoAvailable();
271 }
272
273 private void switchToConversation() {
274 final Contact contact = getWith();
275 final Conversation conversation =
276 xmppConnectionService.findOrCreateConversation(
277 contact.getAccount(), contact.getJid(), false, true);
278 switchToConversation(conversation);
279 }
280
281 private void toggleDialpadVisibility() {
282 if (binding.dialpad.getVisibility() == View.VISIBLE) {
283 binding.dialpad.setVisibility(View.GONE);
284 }
285 else {
286 binding.dialpad.setVisibility(View.VISIBLE);
287 }
288 }
289
290 public boolean onOptionsItemSelected(final MenuItem item) {
291 final var itemItem = item.getItemId();
292 if (itemItem == R.id.action_help) {
293 launchHelpInBrowser();
294 return true;
295 } else if (itemItem == R.id.action_goto_chat) {
296 switchToConversation();
297 return true;
298 } else if (itemItem == R.id.action_dialpad) {
299 toggleDialpadVisibility();
300 return true;
301 } else if (itemItem == R.id.action_switch_to_video) {
302 requestPermissionAndSwitchToVideo();
303 return true;
304 } else {
305 return super.onOptionsItemSelected(item);
306 }
307 }
308
309 private void launchHelpInBrowser() {
310 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
311 try {
312 startActivity(intent);
313 } catch (final ActivityNotFoundException e) {
314 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
315 .show();
316 }
317 }
318
319 private void endCall(View view) {
320 endCall();
321 }
322
323 private void endCall() {
324 if (this.rtpConnectionReference == null) {
325 retractSessionProposal();
326 finish();
327 } else {
328 try {
329 requireRtpConnection().endCall();
330 } catch (final IllegalStateException e) {
331 // No call, already done
332 finish();
333 }
334 }
335 }
336
337 private void retractSessionProposal() {
338 final Intent intent = getIntent();
339 final String action = intent.getAction();
340 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
341 final Account account = extractAccount(intent);
342 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
343 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
344 if (!Intent.ACTION_VIEW.equals(action)
345 || state == null
346 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
347 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
348 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
349 }
350 xmppConnectionService
351 .getJingleConnectionManager()
352 .retractSessionProposal(account, with.asBareJid());
353 }
354
355 private void rejectCall(View view) {
356 requireRtpConnection().rejectCall();
357 finish();
358 }
359
360 private void acceptCall(View view) {
361 requestPermissionsAndAcceptCall();
362 }
363
364 private void acceptContentAdd() {
365 try {
366 requireRtpConnection()
367 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
368 } catch (final IllegalStateException e) {
369 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
370 }
371 }
372
373 private void requestPermissionAndSwitchToVideo() {
374 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
375 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
376 switchToVideo();
377 }
378 }
379
380 private void switchToVideo() {
381 try {
382 requireRtpConnection().addMedia(Media.VIDEO);
383 } catch (final IllegalStateException e) {
384 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
385 }
386 }
387
388 private void acceptContentAdd(final ContentAddition contentAddition) {
389 if (contentAddition == null
390 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
391 Log.d(Config.LOGTAG, "ignore press on content-accept button");
392 return;
393 }
394 requestPermissionAndAcceptContentAdd(contentAddition);
395 }
396
397 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
398 final List<String> permissions = permissions(contentAddition.media());
399 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
400 try {
401 requireRtpConnection().acceptContentAdd(contentAddition.summary);
402 } catch (final IllegalStateException e) {
403 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
404 }
405 }
406 }
407
408 private void rejectContentAdd(final View view) {
409 requireRtpConnection().rejectContentAdd();
410 }
411
412 private void requestPermissionsAndAcceptCall() {
413 final List<String> permissions = permissions(getMedia());
414 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
415 putScreenInCallMode();
416 acceptCall();
417 }
418 }
419
420 private List<String> permissions(final Set<Media> media) {
421 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
422 if (media.contains(Media.VIDEO)) {
423 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
424 } else {
425 permissions.add(Manifest.permission.RECORD_AUDIO);
426 }
427 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
428 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
429 }
430 return permissions.build();
431 }
432
433 private void acceptCall() {
434 try {
435 requireRtpConnection().acceptCall();
436 } catch (final IllegalStateException e) {
437 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
438 }
439 }
440
441 private void putScreenInCallMode() {
442 putScreenInCallMode(requireRtpConnection().getMedia());
443 }
444
445 private void putScreenInCallMode(final Set<Media> media) {
446 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
447 if (Media.audioOnly(media)) {
448 final JingleRtpConnection rtpConnection =
449 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
450 final CallIntegration callIntegration =
451 rtpConnection == null ? null : rtpConnection.getCallIntegration();
452 if (callIntegration == null
453 || callIntegration.getSelectedAudioDevice()
454 == CallIntegration.AudioDevice.EARPIECE) {
455 acquireProximityWakeLock();
456 }
457 }
458 lockOrientation(media);
459 }
460
461 private void lockOrientation(final Set<Media> media) {
462 if (Media.audioOnly(media)) {
463 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
464 } else {
465 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
466 }
467 }
468
469 @SuppressLint("WakelockTimeout")
470 private void acquireProximityWakeLock() {
471 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
472 if (powerManager == null) {
473 Log.e(Config.LOGTAG, "power manager not available");
474 return;
475 }
476 if (isFinishing()) {
477 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
478 return;
479 }
480 if (this.mProximityWakeLock == null) {
481 this.mProximityWakeLock =
482 powerManager.newWakeLock(
483 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
484 }
485 if (!this.mProximityWakeLock.isHeld()) {
486 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
487 this.mProximityWakeLock.acquire();
488 }
489 }
490
491 private void releaseProximityWakeLock() {
492 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
493 Log.d(Config.LOGTAG, "releasing proximity wake lock");
494 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
495 this.mProximityWakeLock = null;
496 }
497 }
498
499 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
500 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
501 acquireProximityWakeLock();
502 } else {
503 releaseProximityWakeLock();
504 }
505 }
506
507 @Override
508 protected void refreshUiReal() {}
509
510 @Override
511 public void onNewIntent(final Intent intent) {
512 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
513 super.onNewIntent(intent);
514 if (intent == null) {
515 return;
516 }
517 setIntent(intent);
518 if (xmppConnectionService == null) {
519 Log.d(
520 Config.LOGTAG,
521 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
522 return;
523 }
524 initializeWithIntent(Event.ON_NEW_INTENT, intent);
525 }
526
527 @Override
528 protected void onBackendConnected() {
529 final var intent = getIntent();
530 if (intent == null) {
531 return;
532 }
533 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
534 }
535
536 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
537 final String action = intent.getAction();
538 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
539 final Account account = extractAccount(intent);
540 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
541 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
542 if (sessionId != null) {
543 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
544 return;
545 }
546 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
547 Log.d(Config.LOGTAG, "intent action was accept");
548 requestPermissionsAndAcceptCall();
549 resetIntent(intent.getExtras());
550 }
551 } else if (Intent.ACTION_VIEW.equals(action)) {
552 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
553 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
554 xmppConnectionService
555 .getJingleConnectionManager()
556 .getTerminalSessionState(with, proposedSessionId);
557 if (terminatedRtpSession != null) {
558 // termination (due to message error or 'busy' was faster than opening the activity
559 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
560 return;
561 }
562 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
563 final RtpEndUserState state =
564 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
565 if (state != null) {
566 Log.d(Config.LOGTAG, "restored last state from intent extra");
567 updateButtonConfiguration(state);
568 updateVerifiedShield(false);
569 updateStateDisplay(state);
570 updateIncomingCallScreen(state);
571 invalidateOptionsMenu();
572 }
573 setWith(account.getRoster().getContact(with), state);
574 if (xmppConnectionService
575 .getJingleConnectionManager()
576 .fireJingleRtpConnectionStateUpdates()) {
577 return;
578 }
579 if (END_CARD.contains(state)) {
580 return;
581 }
582 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
583 final Set<Media> media = actionToMedia(lastAction);
584 if (xmppConnectionService
585 .getJingleConnectionManager()
586 .hasMatchingProposal(account, with)) {
587 putScreenInCallMode(media);
588 return;
589 }
590 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
591 finish();
592 }
593 }
594
595 private void setWith(final RtpEndUserState state) {
596 setWith(getWith(), state);
597 }
598
599 private void setWith(final Contact contact, final RtpEndUserState state) {
600 binding.with.setText(contact.getDisplayName());
601 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
602 .contains(state)) {
603 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
604 binding.withJid.setVisibility(View.VISIBLE);
605 } else {
606 binding.withJid.setVisibility(View.GONE);
607 }
608 }
609
610 @Override
611 public void onRequestPermissionsResult(
612 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
613 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
614 final PermissionUtils.PermissionResult permissionResult =
615 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
616 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
617 if (requestCode == REQUEST_ACCEPT_CALL) {
618 acceptCall();
619 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
620 acceptContentAdd();
621 } else if (requestCode == REQUEST_ADD_CONTENT) {
622 switchToVideo();
623 }
624 } else {
625 @StringRes int res;
626 final String firstDenied =
627 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
628 if (firstDenied == null) {
629 return;
630 }
631 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
632 res = R.string.no_microphone_permission;
633 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
634 res = R.string.no_camera_permission;
635 } else {
636 throw new IllegalStateException("Invalid permission result request");
637 }
638 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
639 .show();
640 }
641 }
642
643 @Override
644 public void onStart() {
645 super.onStart();
646 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
647 this.binding.remoteVideo.setOnAspectRatioChanged(this);
648 }
649
650 @Override
651 public void onStop() {
652 mHandler.removeCallbacks(mTickExecutor);
653 binding.remoteVideo.release();
654 binding.remoteVideo.setOnAspectRatioChanged(null);
655 binding.localVideo.release();
656 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
657 final JingleRtpConnection jingleRtpConnection =
658 weakReference == null ? null : weakReference.get();
659 if (jingleRtpConnection != null) {
660 releaseVideoTracks(jingleRtpConnection);
661 }
662 releaseProximityWakeLock();
663 super.onStop();
664 }
665
666 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
667 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
668 if (remoteVideo.isPresent()) {
669 remoteVideo.get().removeSink(binding.remoteVideo);
670 }
671 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
672 if (localVideo.isPresent()) {
673 localVideo.get().removeSink(binding.localVideo);
674 }
675 }
676
677 @Override
678 public void onBackPressed() {
679 if (isConnected()) {
680 if (switchToPictureInPicture()) {
681 return;
682 }
683 } else {
684 endCall();
685 }
686 super.onBackPressed();
687 }
688
689 @Override
690 public void onUserLeaveHint() {
691 super.onUserLeaveHint();
692 if (switchToPictureInPicture()) {
693 return;
694 }
695 // TODO apparently this method is not getting called on Android 10 when using the task
696 // switcher
697 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
698 retractSessionProposal();
699 }
700 }
701
702 private boolean isConnected() {
703 final JingleRtpConnection connection =
704 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
705 final RtpEndUserState endUserState =
706 connection == null ? null : connection.getEndUserState();
707 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
708 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
709 }
710
711 private boolean switchToPictureInPicture() {
712 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
713 if (shouldBePictureInPicture()) {
714 startPictureInPicture();
715 return true;
716 }
717 }
718 return false;
719 }
720
721 @RequiresApi(api = Build.VERSION_CODES.O)
722 private void startPictureInPicture() {
723 try {
724 final Rational rational = this.binding.remoteVideo.getAspectRatio();
725 final Rational clippedRational = Rationals.clip(rational);
726 Log.d(
727 Config.LOGTAG,
728 "suggested rational " + rational + ". clipped to " + clippedRational);
729 enterPictureInPictureMode(
730 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
731 } catch (final IllegalStateException e) {
732 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
733 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
734 }
735 }
736
737 @Override
738 public void onAspectRatioChanged(final Rational rational) {
739 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
740 final Rational clippedRational = Rationals.clip(rational);
741 Log.d(
742 Config.LOGTAG,
743 "suggested rational after aspect ratio change "
744 + rational
745 + ". clipped to "
746 + clippedRational);
747 setPictureInPictureParams(
748 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
749 }
750 }
751
752 private boolean deviceSupportsPictureInPicture() {
753 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
754 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
755 } else {
756 return false;
757 }
758 }
759
760 private boolean shouldBePictureInPicture() {
761 try {
762 final JingleRtpConnection rtpConnection = requireRtpConnection();
763 return rtpConnection.getMedia().contains(Media.VIDEO)
764 && Arrays.asList(
765 RtpEndUserState.ACCEPTING_CALL,
766 RtpEndUserState.CONNECTING,
767 RtpEndUserState.CONNECTED)
768 .contains(rtpConnection.getEndUserState());
769 } catch (final IllegalStateException e) {
770 return false;
771 }
772 }
773
774 private boolean initializeActivityWithRunningRtpSession(
775 final Account account, Jid with, String sessionId) {
776 final WeakReference<JingleRtpConnection> reference =
777 xmppConnectionService
778 .getJingleConnectionManager()
779 .findJingleRtpConnection(account, with, sessionId);
780 if (reference == null || reference.get() == null) {
781 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
782 xmppConnectionService
783 .getJingleConnectionManager()
784 .getTerminalSessionState(with, sessionId);
785 if (terminatedRtpSession == null) {
786 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
787 finish();
788 return true;
789 }
790 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
791 return true;
792 }
793 this.rtpConnectionReference = reference;
794 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
795 final boolean verified = requireRtpConnection().isVerified();
796 if (currentState == RtpEndUserState.ENDED) {
797 finish();
798 return true;
799 }
800 final Set<Media> media = getMedia();
801 final ContentAddition contentAddition = getPendingContentAddition();
802 if (currentState == RtpEndUserState.INCOMING_CALL) {
803 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
804 }
805 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
806 requireRtpConnection().getState())) {
807 putScreenInCallMode();
808 }
809 setWith(currentState);
810 updateVideoViews(currentState);
811 updateStateDisplay(currentState, media, contentAddition);
812 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
813 updateButtonConfiguration(currentState, media, contentAddition);
814 updateIncomingCallScreen(currentState);
815 invalidateOptionsMenu();
816 return false;
817 }
818
819 private void initializeWithTerminatedSessionState(
820 final Account account,
821 final Jid with,
822 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
823 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
824 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
825 finish();
826 return;
827 }
828 final RtpEndUserState state = terminatedRtpSession.state;
829 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
830 updateButtonConfiguration(state);
831 updateStateDisplay(state);
832 updateIncomingCallScreen(state);
833 updateCallDuration();
834 updateVerifiedShield(false);
835 invalidateOptionsMenu();
836 setWith(account.getRoster().getContact(with), state);
837 }
838
839 private void reInitializeActivityWithRunningRtpSession(
840 final Account account, Jid with, String sessionId) {
841 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
842 resetIntent(account, with, sessionId);
843 }
844
845 private void resetIntent(final Account account, final Jid with, final String sessionId) {
846 final Intent intent = new Intent(Intent.ACTION_VIEW);
847 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
848 intent.putExtra(EXTRA_WITH, with.toEscapedString());
849 intent.putExtra(EXTRA_SESSION_ID, sessionId);
850 setIntent(intent);
851 }
852
853 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
854 surfaceViewRenderer.setVisibility(View.VISIBLE);
855 try {
856 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
857 } catch (final IllegalStateException ignored) {
858 // SurfaceViewRenderer was already initialized
859 } catch (final RuntimeException e) {
860 if (Throwables.getRootCause(e) instanceof GLException glException) {
861 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
862 }
863 }
864 surfaceViewRenderer.setEnableHardwareScaler(true);
865 }
866
867 private void updateStateDisplay(final RtpEndUserState state) {
868 updateStateDisplay(state, Collections.emptySet(), null);
869 }
870
871 private void updateStateDisplay(
872 final RtpEndUserState state,
873 final Set<Media> media,
874 final ContentAddition contentAddition) {
875 switch (state) {
876 case INCOMING_CALL -> {
877 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
878 if (media.contains(Media.VIDEO)) {
879 setTitle(R.string.rtp_state_incoming_video_call);
880 } else {
881 setTitle(R.string.rtp_state_incoming_call);
882 }
883 }
884 case INCOMING_CONTENT_ADD -> {
885 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
886 setTitle(R.string.rtp_state_content_add_video);
887 } else {
888 setTitle(R.string.rtp_state_content_add);
889 }
890 }
891 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
892 case CONNECTED -> setTitle(R.string.rtp_state_connected);
893 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
894 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
895 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
896 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
897 case RINGING -> setTitle(R.string.rtp_state_ringing);
898 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
899 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
900 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
901 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
902 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
903 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
904 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
905 case ENDED -> throw new IllegalStateException(
906 "Activity should have called finishAndReleaseWakeLock();");
907 default -> throw new IllegalStateException(
908 String.format("State %s has not been handled in UI", state));
909 }
910 }
911
912 private void updateVerifiedShield(final boolean verified) {
913 if (isPictureInPicture()) {
914 this.binding.verified.setVisibility(View.GONE);
915 return;
916 }
917 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
918 }
919
920 private void updateIncomingCallScreen(final RtpEndUserState state) {
921 updateIncomingCallScreen(state, null);
922 }
923
924 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
925 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
926 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
927 if (show) {
928 binding.contactPhoto.setVisibility(View.VISIBLE);
929 if (contact == null) {
930 AvatarWorkerTask.loadAvatar(
931 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
932 } else {
933 AvatarWorkerTask.loadAvatar(
934 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
935 }
936 } else {
937 binding.contactPhoto.setVisibility(View.GONE);
938 }
939 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
940 binding.usingAccount.setVisibility(View.VISIBLE);
941 binding.usingAccount.setText(
942 getString(
943 R.string.using_account,
944 account.getJid().asBareJid().toEscapedString()));
945 } else {
946 binding.usingAccount.setVisibility(View.GONE);
947 binding.contactPhoto.setVisibility(View.GONE);
948 }
949 }
950
951 private Set<Media> getMedia() {
952 return requireRtpConnection().getMedia();
953 }
954
955 public ContentAddition getPendingContentAddition() {
956 return requireRtpConnection().getPendingContentAddition();
957 }
958
959 private void updateButtonConfiguration(final RtpEndUserState state) {
960 updateButtonConfiguration(state, Collections.emptySet(), null);
961 }
962
963 @SuppressLint("RestrictedApi")
964 private void updateButtonConfiguration(
965 final RtpEndUserState state,
966 final Set<Media> media,
967 final ContentAddition contentAddition) {
968 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
969 this.binding.rejectCall.setVisibility(View.INVISIBLE);
970 this.binding.endCall.setVisibility(View.INVISIBLE);
971 this.binding.acceptCall.setVisibility(View.INVISIBLE);
972 } else if (state == RtpEndUserState.INCOMING_CALL) {
973 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
974 this.binding.rejectCall.setOnClickListener(this::rejectCall);
975 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
976 this.binding.rejectCall.setVisibility(View.VISIBLE);
977 this.binding.endCall.setVisibility(View.INVISIBLE);
978 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
979 this.binding.acceptCall.setOnClickListener(this::acceptCall);
980 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
981 this.binding.acceptCall.setVisibility(View.VISIBLE);
982 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
983 this.binding.rejectCall.setContentDescription(
984 getString(R.string.reject_switch_to_video));
985 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
986 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
987 this.binding.rejectCall.setVisibility(View.VISIBLE);
988 this.binding.endCall.setVisibility(View.INVISIBLE);
989 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
990 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
991 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
992 this.binding.acceptCall.setVisibility(View.VISIBLE);
993 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
994 .contains(state)) {
995 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
996 this.binding.rejectCall.setOnClickListener(this::exit);
997 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
998 this.binding.rejectCall.setVisibility(View.VISIBLE);
999 this.binding.endCall.setVisibility(View.INVISIBLE);
1000 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1001 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1002 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
1003 this.binding.acceptCall.setVisibility(View.VISIBLE);
1004 } else if (asList(
1005 RtpEndUserState.CONNECTIVITY_ERROR,
1006 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1007 RtpEndUserState.APPLICATION_ERROR,
1008 RtpEndUserState.RETRACTED,
1009 RtpEndUserState.SECURITY_ERROR)
1010 .contains(state)) {
1011 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1012 this.binding.rejectCall.setOnClickListener(this::exit);
1013 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1014 this.binding.rejectCall.setVisibility(View.VISIBLE);
1015 this.binding.endCall.setVisibility(View.INVISIBLE);
1016 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1017 this.binding.acceptCall.setOnClickListener(this::retry);
1018 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1019 this.binding.acceptCall.setVisibility(View.VISIBLE);
1020 } else {
1021 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1022 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1023 this.binding.endCall.setOnClickListener(this::endCall);
1024 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1025 this.binding.endCall.setVisibility(View.VISIBLE);
1026 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1027 }
1028 updateInCallButtonConfiguration(state, media);
1029 }
1030
1031 private boolean isPictureInPicture() {
1032 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1033 return isInPictureInPictureMode();
1034 } else {
1035 return false;
1036 }
1037 }
1038
1039 private void updateInCallButtonConfiguration() {
1040 updateInCallButtonConfiguration(
1041 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1042 }
1043
1044 @SuppressLint("RestrictedApi")
1045 private void updateInCallButtonConfiguration(
1046 final RtpEndUserState state, final Set<Media> media) {
1047 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1048 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1049 if (media.contains(Media.VIDEO)) {
1050 final JingleRtpConnection rtpConnection = requireRtpConnection();
1051 updateInCallButtonConfigurationVideo(
1052 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1053 } else {
1054 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1055 updateInCallButtonConfigurationSpeaker(
1056 callIntegration.getSelectedAudioDevice(),
1057 callIntegration.getAudioDevices().size());
1058 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1059 }
1060 if (media.contains(Media.AUDIO)) {
1061 updateInCallButtonConfigurationMicrophone(
1062 requireRtpConnection().isMicrophoneEnabled());
1063 } else {
1064 this.binding.inCallActionLeft.setVisibility(View.GONE);
1065 }
1066 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1067 && !isPictureInPicture()
1068 && Media.audioOnly(media)) {
1069 final CallIntegration callIntegration;
1070 try {
1071 callIntegration = requireCallIntegration();
1072 } catch (final IllegalStateException e) {
1073 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1074 return;
1075 }
1076 updateInCallButtonConfigurationSpeaker(
1077 callIntegration.getSelectedAudioDevice(),
1078 callIntegration.getAudioDevices().size());
1079 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1080 } else {
1081 this.binding.inCallActionLeft.setVisibility(View.GONE);
1082 this.binding.inCallActionRight.setVisibility(View.GONE);
1083 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1084 }
1085 }
1086
1087 @SuppressLint("RestrictedApi")
1088 private void updateInCallButtonConfigurationSpeaker(
1089 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1090 switch (selectedAudioDevice) {
1091 case EARPIECE -> {
1092 this.binding.inCallActionRight.setImageResource(
1093 R.drawable.ic_volume_off_24dp);
1094 if (numberOfChoices >= 2) {
1095 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1096 } else {
1097 this.binding.inCallActionRight.setOnClickListener(null);
1098 this.binding.inCallActionRight.setClickable(false);
1099 }
1100 }
1101 case WIRED_HEADSET -> {
1102 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1103 this.binding.inCallActionRight.setOnClickListener(null);
1104 this.binding.inCallActionRight.setClickable(false);
1105 }
1106 case SPEAKER_PHONE -> {
1107 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1108 if (numberOfChoices >= 2) {
1109 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1110 } else {
1111 this.binding.inCallActionRight.setOnClickListener(null);
1112 this.binding.inCallActionRight.setClickable(false);
1113 }
1114 }
1115 case BLUETOOTH -> {
1116 this.binding.inCallActionRight.setImageResource(
1117 R.drawable.ic_bluetooth_audio_24dp);
1118 this.binding.inCallActionRight.setOnClickListener(null);
1119 this.binding.inCallActionRight.setClickable(false);
1120 }
1121 }
1122 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1123 }
1124
1125 @SuppressLint("RestrictedApi")
1126 private void updateInCallButtonConfigurationVideo(
1127 final boolean videoEnabled, final boolean isCameraSwitchable) {
1128 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1129 if (isCameraSwitchable) {
1130 this.binding.inCallActionFarRight.setImageResource(
1131 R.drawable.ic_flip_camera_android_24dp);
1132 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1133 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1134 } else {
1135 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1136 }
1137 if (videoEnabled) {
1138 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1139 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1140 } else {
1141 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1142 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1143 }
1144 }
1145
1146 private void switchCamera(final View view) {
1147 Futures.addCallback(
1148 requireRtpConnection().switchCamera(),
1149 new FutureCallback<>() {
1150 @Override
1151 public void onSuccess(@Nullable Boolean isFrontCamera) {
1152 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1153 }
1154
1155 @Override
1156 public void onFailure(@NonNull final Throwable throwable) {
1157 Log.d(
1158 Config.LOGTAG,
1159 "could not switch camera",
1160 Throwables.getRootCause(throwable));
1161 Toast.makeText(
1162 RtpSessionActivity.this,
1163 R.string.could_not_switch_camera,
1164 Toast.LENGTH_LONG)
1165 .show();
1166 }
1167 },
1168 MainThreadExecutor.getInstance());
1169 }
1170
1171 private void enableVideo(View view) {
1172 try {
1173 requireRtpConnection().setVideoEnabled(true);
1174 } catch (final IllegalStateException e) {
1175 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1176 return;
1177 }
1178 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1179 }
1180
1181 private void disableVideo(View view) {
1182 final JingleRtpConnection rtpConnection = requireRtpConnection();
1183 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1184 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1185 rtpConnection.retractContentAdd();
1186 return;
1187 }
1188 requireRtpConnection().setVideoEnabled(false);
1189 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1190 }
1191
1192 @SuppressLint("RestrictedApi")
1193 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1194 if (microphoneEnabled) {
1195 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1196 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1197 } else {
1198 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1199 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1200 }
1201 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1202 }
1203
1204 private void updateCallDuration() {
1205 final JingleRtpConnection connection =
1206 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1207 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1208 this.binding.duration.setVisibility(View.GONE);
1209 return;
1210 }
1211 if (connection.zeroDuration()) {
1212 this.binding.duration.setVisibility(View.GONE);
1213 } else {
1214 this.binding.duration.setText(
1215 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1216 this.binding.duration.setVisibility(View.VISIBLE);
1217 }
1218 }
1219
1220 private void updateVideoViews(final RtpEndUserState state) {
1221 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1222 binding.localVideo.setVisibility(View.GONE);
1223 binding.localVideo.release();
1224 binding.remoteVideoWrapper.setVisibility(View.GONE);
1225 binding.remoteVideo.release();
1226 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1227 if (isPictureInPicture()) {
1228 binding.appBarLayout.setVisibility(View.GONE);
1229 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1230 if (Arrays.asList(
1231 RtpEndUserState.APPLICATION_ERROR,
1232 RtpEndUserState.CONNECTIVITY_ERROR,
1233 RtpEndUserState.SECURITY_ERROR)
1234 .contains(state)) {
1235 binding.pipWarning.setVisibility(View.VISIBLE);
1236 binding.pipWaiting.setVisibility(View.GONE);
1237 } else {
1238 binding.pipWarning.setVisibility(View.GONE);
1239 binding.pipWaiting.setVisibility(View.GONE);
1240 }
1241 } else {
1242 binding.appBarLayout.setVisibility(View.VISIBLE);
1243 binding.pipPlaceholder.setVisibility(View.GONE);
1244 }
1245 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1246 return;
1247 }
1248 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1249 binding.localVideo.setVisibility(View.GONE);
1250 binding.remoteVideoWrapper.setVisibility(View.GONE);
1251 binding.appBarLayout.setVisibility(View.GONE);
1252 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1253 binding.pipWarning.setVisibility(View.GONE);
1254 binding.pipWaiting.setVisibility(View.VISIBLE);
1255 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1256 return;
1257 }
1258 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1259 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1260 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1261 // paint local view over remote view
1262 binding.localVideo.setZOrderMediaOverlay(true);
1263 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1264 addSink(localVideoTrack.get(), binding.localVideo);
1265 } else {
1266 binding.localVideo.setVisibility(View.GONE);
1267 }
1268 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1269 if (remoteVideoTrack.isPresent()) {
1270 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1271 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1272 binding.remoteVideo.setScalingType(
1273 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1274 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1275 if (state == RtpEndUserState.CONNECTED) {
1276 binding.appBarLayout.setVisibility(View.GONE);
1277 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1278 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1279 } else {
1280 binding.appBarLayout.setVisibility(View.VISIBLE);
1281 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1282 binding.remoteVideoWrapper.setVisibility(View.GONE);
1283 }
1284 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1285 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1286 } else {
1287 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1288 }
1289 } else {
1290 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1291 binding.remoteVideoWrapper.setVisibility(View.GONE);
1292 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1293 }
1294 }
1295
1296 private Optional<VideoTrack> getLocalVideoTrack() {
1297 final JingleRtpConnection connection =
1298 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1299 if (connection == null) {
1300 return Optional.absent();
1301 }
1302 return connection.getLocalVideoTrack();
1303 }
1304
1305 private Optional<VideoTrack> getRemoteVideoTrack() {
1306 final JingleRtpConnection connection =
1307 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1308 if (connection == null) {
1309 return Optional.absent();
1310 }
1311 return connection.getRemoteVideoTrack();
1312 }
1313
1314 private void disableMicrophone(View view) {
1315 final JingleRtpConnection rtpConnection = requireRtpConnection();
1316 if (rtpConnection.setMicrophoneEnabled(false)) {
1317 updateInCallButtonConfiguration();
1318 }
1319 }
1320
1321 private void enableMicrophone(View view) {
1322 final JingleRtpConnection rtpConnection = requireRtpConnection();
1323 if (rtpConnection.setMicrophoneEnabled(true)) {
1324 updateInCallButtonConfiguration();
1325 }
1326 }
1327
1328 private void switchToEarpiece(final View view) {
1329 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1330 acquireProximityWakeLock();
1331 }
1332
1333 private void switchToSpeaker(final View view) {
1334 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1335 releaseProximityWakeLock();
1336 }
1337
1338 private void retry(final View view) {
1339 final Intent intent = getIntent();
1340 final Account account = extractAccount(intent);
1341 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1342 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1343 final String action = intent.getAction();
1344 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1345 this.rtpConnectionReference = null;
1346 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1347 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1348 }
1349
1350 private void exit(final View view) {
1351 finish();
1352 }
1353
1354 private void recordVoiceMail(final View view) {
1355 final Intent intent = getIntent();
1356 final Account account = extractAccount(intent);
1357 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1358 final Conversation conversation =
1359 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1360 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1361 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1362 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1363 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1364 launchIntent.putExtra(
1365 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1366 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1367 startActivity(launchIntent);
1368 finish();
1369 }
1370
1371 private Contact getWith() {
1372 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1373 final Account account = id.account;
1374 return account.getRoster().getContact(id.with);
1375 }
1376
1377 private JingleRtpConnection requireRtpConnection() {
1378 final JingleRtpConnection connection =
1379 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1380 if (connection == null) {
1381 throw new IllegalStateException("No RTP connection found");
1382 }
1383 return connection;
1384 }
1385
1386 private CallIntegration requireCallIntegration() {
1387 return requireOngoingRtpSession().getCallIntegration();
1388 }
1389
1390 private OngoingRtpSession requireOngoingRtpSession() {
1391 final JingleRtpConnection connection =
1392 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1393 if (connection != null) {
1394 return connection;
1395 }
1396 final Intent currentIntent = getIntent();
1397 final String withExtra =
1398 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1399 final var account = extractAccount(currentIntent);
1400 if (withExtra == null) {
1401 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1402 }
1403 final var matching =
1404 xmppConnectionService
1405 .getJingleConnectionManager()
1406 .matchingProposal(account, Jid.of(withExtra));
1407 if (matching.isPresent()) {
1408 return matching.get();
1409 }
1410 throw new IllegalStateException("No matching session proposal");
1411 }
1412
1413 @Override
1414 public void onJingleRtpConnectionUpdate(
1415 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1416 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1417 if (END_CARD.contains(state)) {
1418 Log.d(Config.LOGTAG, "end card reached");
1419 releaseProximityWakeLock();
1420 runOnUiThread(
1421 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1422 }
1423 if (with.isBareJid()) {
1424 updateRtpSessionProposalState(account, with, state);
1425 return;
1426 }
1427 if (emptyReference(this.rtpConnectionReference)) {
1428 if (END_CARD.contains(state)) {
1429 Log.d(Config.LOGTAG, "not reinitializing session");
1430 return;
1431 }
1432 // this happens when going from proposed session to actual session
1433 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1434 return;
1435 }
1436 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1437 final boolean verified = requireRtpConnection().isVerified();
1438 final Set<Media> media = getMedia();
1439 lockOrientation(media);
1440 final ContentAddition contentAddition = getPendingContentAddition();
1441 final Contact contact = getWith();
1442 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1443 if (state == RtpEndUserState.ENDED) {
1444 finish();
1445 return;
1446 }
1447 runOnUiThread(
1448 () -> {
1449 updateStateDisplay(state, media, contentAddition);
1450 updateVerifiedShield(
1451 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1452 updateButtonConfiguration(state, media, contentAddition);
1453 updateVideoViews(state);
1454 updateIncomingCallScreen(state, contact);
1455 invalidateOptionsMenu();
1456 });
1457 if (END_CARD.contains(state)) {
1458 final JingleRtpConnection rtpConnection = requireRtpConnection();
1459 resetIntent(account, with, state, rtpConnection.getMedia());
1460 releaseVideoTracks(rtpConnection);
1461 this.rtpConnectionReference = null;
1462 }
1463 } else {
1464 Log.d(Config.LOGTAG, "received update for other rtp session");
1465 }
1466 }
1467
1468 @Override
1469 public void onAudioDeviceChanged(
1470 final CallIntegration.AudioDevice selectedAudioDevice,
1471 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1472 Log.d(
1473 Config.LOGTAG,
1474 "onAudioDeviceChanged in activity: selected:"
1475 + selectedAudioDevice
1476 + ", available:"
1477 + availableAudioDevices);
1478 try {
1479 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1480 final RtpEndUserState endUserState;
1481 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1482 endUserState = jingleRtpConnection.getEndUserState();
1483 } else {
1484 // for session proposals all end user states are functionally the same
1485 endUserState = RtpEndUserState.RINGING;
1486 }
1487 final Set<Media> media = ongoingRtpSession.getMedia();
1488 if (END_CARD.contains(endUserState)) {
1489 Log.d(
1490 Config.LOGTAG,
1491 "onAudioDeviceChanged() nothing to do because end card has been reached");
1492 } else {
1493 if (Media.audioOnly(media)
1494 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1495 final CallIntegration callIntegration = requireCallIntegration();
1496 updateInCallButtonConfigurationSpeaker(
1497 callIntegration.getSelectedAudioDevice(),
1498 callIntegration.getAudioDevices().size());
1499 }
1500 Log.d(
1501 Config.LOGTAG,
1502 "put proximity wake lock into proper state after device update");
1503 putProximityWakeLockInProperState(selectedAudioDevice);
1504 }
1505 } catch (final IllegalStateException e) {
1506 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1507 }
1508 }
1509
1510 @Override
1511 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1512 super.onSaveInstanceState(outState);
1513 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1514 }
1515
1516 private void updateRtpSessionProposalState(
1517 final Account account, final Jid with, final RtpEndUserState state) {
1518 final Intent currentIntent = getIntent();
1519 final String withExtra =
1520 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1521 if (withExtra == null) {
1522 return;
1523 }
1524 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1525 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1526 runOnUiThread(
1527 () -> {
1528 updateVerifiedShield(false);
1529 updateStateDisplay(state);
1530 updateButtonConfiguration(state, media, null);
1531 updateIncomingCallScreen(state);
1532 invalidateOptionsMenu();
1533 });
1534 resetIntent(account, with, state, media);
1535 }
1536 }
1537
1538 private void resetIntent(final Bundle extras) {
1539 final Intent intent = new Intent(Intent.ACTION_VIEW);
1540 intent.putExtras(extras);
1541 setIntent(intent);
1542 }
1543
1544 private void resetIntent(
1545 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1546 final Intent intent = new Intent(Intent.ACTION_VIEW);
1547 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1548 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1549 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1550 } else {
1551 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1552 }
1553 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1554 intent.putExtra(
1555 EXTRA_LAST_ACTION,
1556 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1557 setIntent(intent);
1558 }
1559
1560 private static boolean emptyReference(final WeakReference<?> weakReference) {
1561 return weakReference == null || weakReference.get() == null;
1562 }
1563
1564 private enum Event {
1565 ON_BACKEND_CONNECTED,
1566 ON_NEW_INTENT
1567 }
1568}