1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import eu.siacs.conversations.Config;
44import eu.siacs.conversations.R;
45import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
46import eu.siacs.conversations.entities.Account;
47import eu.siacs.conversations.entities.Contact;
48import eu.siacs.conversations.entities.Conversation;
49import eu.siacs.conversations.services.CallIntegration;
50import eu.siacs.conversations.services.CallIntegrationConnectionService;
51import eu.siacs.conversations.services.XmppConnectionService;
52import eu.siacs.conversations.ui.util.AvatarWorkerTask;
53import eu.siacs.conversations.ui.util.MainThreadExecutor;
54import eu.siacs.conversations.ui.util.Rationals;
55import eu.siacs.conversations.utils.PermissionUtils;
56import eu.siacs.conversations.utils.TimeFrameUtils;
57import eu.siacs.conversations.xmpp.Jid;
58import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
59import eu.siacs.conversations.xmpp.jingle.ContentAddition;
60import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
61import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
62import eu.siacs.conversations.xmpp.jingle.Media;
63import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
64import eu.siacs.conversations.xmpp.jingle.RtpCapability;
65import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
66
67import org.webrtc.RendererCommon;
68import org.webrtc.SurfaceViewRenderer;
69import org.webrtc.VideoTrack;
70
71import java.lang.ref.WeakReference;
72import java.util.Arrays;
73import java.util.Collections;
74import java.util.List;
75import java.util.Set;
76
77public class RtpSessionActivity extends XmppActivity
78 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
79 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
80
81 public static final String EXTRA_WITH = "with";
82 public static final String EXTRA_SESSION_ID = "session_id";
83 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
84 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
85 public static final String EXTRA_LAST_ACTION = "last_action";
86 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
87 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
88 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
89
90 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
91
92 private static final List<RtpEndUserState> END_CARD =
93 Arrays.asList(
94 RtpEndUserState.APPLICATION_ERROR,
95 RtpEndUserState.SECURITY_ERROR,
96 RtpEndUserState.DECLINED_OR_BUSY,
97 RtpEndUserState.CONTACT_OFFLINE,
98 RtpEndUserState.CONNECTIVITY_ERROR,
99 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
100 RtpEndUserState.RETRACTED);
101 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
102 Arrays.asList(
103 RtpEndUserState.APPLICATION_ERROR,
104 RtpEndUserState.CONNECTIVITY_ERROR,
105 RtpEndUserState.SECURITY_ERROR);
106 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
107 Arrays.asList(
108 RtpEndUserState.CONNECTING,
109 RtpEndUserState.CONNECTED,
110 RtpEndUserState.RECONNECTING,
111 RtpEndUserState.INCOMING_CONTENT_ADD);
112 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
113 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
114 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
115 Arrays.asList(
116 RtpEndUserState.ACCEPTING_CALL,
117 RtpEndUserState.CONNECTING,
118 RtpEndUserState.RECONNECTING);
119 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
120 new ImmutableList.Builder<RtpEndUserState>()
121 .add(RtpEndUserState.FINDING_DEVICE)
122 .add(RtpEndUserState.RINGING)
123 .add(RtpEndUserState.ACCEPTING_CALL)
124 .add(RtpEndUserState.CONNECTING)
125 .addAll(STATES_CONSIDERED_CONNECTED)
126 .build();
127 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
128 private static final int REQUEST_ACCEPT_CALL = 0x1111;
129 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
130 private static final int REQUEST_ADD_CONTENT = 0x1113;
131 private WeakReference<JingleRtpConnection> rtpConnectionReference;
132
133 private ActivityRtpSessionBinding binding;
134 private PowerManager.WakeLock mProximityWakeLock;
135
136 private final Handler mHandler = new Handler();
137 private final Runnable mTickExecutor =
138 new Runnable() {
139 @Override
140 public void run() {
141 updateCallDuration();
142 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
143 }
144 };
145
146 public static Set<Media> actionToMedia(final String action) {
147 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
148 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
149 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
150 return ImmutableSet.of(Media.AUDIO);
151 } else {
152 Log.w(
153 Config.LOGTAG,
154 "actionToMedia can not get media set from unknown action " + action);
155 return Collections.emptySet();
156 }
157 }
158
159 private static void addSink(
160 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
161 try {
162 videoTrack.addSink(surfaceViewRenderer);
163 } catch (final IllegalStateException e) {
164 Log.e(
165 Config.LOGTAG,
166 "possible race condition on trying to display video track. ignoring",
167 e);
168 }
169 }
170
171 @Override
172 public void onCreate(Bundle savedInstanceState) {
173 super.onCreate(savedInstanceState);
174 getWindow()
175 .addFlags(
176 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
177 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
178 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
179 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
180 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
181 setSupportActionBar(binding.toolbar);
182 }
183
184 @Override
185 public boolean onCreateOptionsMenu(final Menu menu) {
186 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
187 final MenuItem help = menu.findItem(R.id.action_help);
188 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
189 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
190 help.setVisible(Config.HELP != null && isHelpButtonVisible());
191 gotoChat.setVisible(isSwitchToConversationVisible());
192 switchToVideo.setVisible(isSwitchToVideoVisible());
193 return super.onCreateOptionsMenu(menu);
194 }
195
196 @Override
197 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
198 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
199 if (xmppConnectionService != null) {
200 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
201 return true;
202 }
203 }
204 }
205 return super.onKeyDown(keyCode, event);
206 }
207
208 private boolean isHelpButtonVisible() {
209 try {
210 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
211 } catch (IllegalStateException e) {
212 final Intent intent = getIntent();
213 final String state =
214 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
215 if (state != null) {
216 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
217 } else {
218 return false;
219 }
220 }
221 }
222
223 private boolean isSwitchToConversationVisible() {
224 final JingleRtpConnection connection =
225 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
226 return connection != null
227 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
228 }
229
230 private boolean isSwitchToVideoVisible() {
231 final JingleRtpConnection connection =
232 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
233 if (connection == null) {
234 return false;
235 }
236 return connection.isSwitchToVideoAvailable();
237 }
238
239 private void switchToConversation() {
240 final Contact contact = getWith();
241 final Conversation conversation =
242 xmppConnectionService.findOrCreateConversation(
243 contact.getAccount(), contact.getJid(), false, true);
244 switchToConversation(conversation);
245 }
246
247 public boolean onOptionsItemSelected(final MenuItem item) {
248 final var itemItem = item.getItemId();
249 if (itemItem == R.id.action_help) {
250 launchHelpInBrowser();
251 return true;
252 } else if (itemItem == R.id.action_goto_chat) {
253 switchToConversation();
254 return true;
255 } else if (itemItem == R.id.action_switch_to_video) {
256 requestPermissionAndSwitchToVideo();
257 return true;
258 } else {
259 return super.onOptionsItemSelected(item);
260 }
261 }
262
263 private void launchHelpInBrowser() {
264 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
265 try {
266 startActivity(intent);
267 } catch (final ActivityNotFoundException e) {
268 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
269 .show();
270 }
271 }
272
273 private void endCall(View view) {
274 endCall();
275 }
276
277 private void endCall() {
278 if (this.rtpConnectionReference == null) {
279 retractSessionProposal();
280 finish();
281 } else {
282 requireRtpConnection().endCall();
283 }
284 }
285
286 private void retractSessionProposal() {
287 final Intent intent = getIntent();
288 final String action = intent.getAction();
289 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
290 final Account account = extractAccount(intent);
291 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
292 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
293 if (!Intent.ACTION_VIEW.equals(action)
294 || state == null
295 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
296 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
297 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
298 }
299 xmppConnectionService
300 .getJingleConnectionManager()
301 .retractSessionProposal(account, with.asBareJid());
302 }
303
304 private void rejectCall(View view) {
305 requireRtpConnection().rejectCall();
306 finish();
307 }
308
309 private void acceptCall(View view) {
310 requestPermissionsAndAcceptCall();
311 }
312
313 private void acceptContentAdd() {
314 try {
315 requireRtpConnection()
316 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
317 } catch (final IllegalStateException e) {
318 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
319 }
320 }
321
322 private void requestPermissionAndSwitchToVideo() {
323 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
324 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
325 switchToVideo();
326 }
327 }
328
329 private void switchToVideo() {
330 try {
331 requireRtpConnection().addMedia(Media.VIDEO);
332 } catch (final IllegalStateException e) {
333 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
334 }
335 }
336
337 private void acceptContentAdd(final ContentAddition contentAddition) {
338 if (contentAddition == null
339 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
340 Log.d(Config.LOGTAG, "ignore press on content-accept button");
341 return;
342 }
343 requestPermissionAndAcceptContentAdd(contentAddition);
344 }
345
346 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
347 final List<String> permissions = permissions(contentAddition.media());
348 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
349 try {
350 requireRtpConnection().acceptContentAdd(contentAddition.summary);
351 } catch (final IllegalStateException e) {
352 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
353 }
354 }
355 }
356
357 private void rejectContentAdd(final View view) {
358 requireRtpConnection().rejectContentAdd();
359 }
360
361 private void requestPermissionsAndAcceptCall() {
362 final List<String> permissions = permissions(getMedia());
363 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
364 putScreenInCallMode();
365 acceptCall();
366 }
367 }
368
369 private List<String> permissions(final Set<Media> media) {
370 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
371 if (media.contains(Media.VIDEO)) {
372 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
373 } else {
374 permissions.add(Manifest.permission.RECORD_AUDIO);
375 }
376 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
377 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
378 }
379 return permissions.build();
380 }
381
382 private void acceptCall() {
383 try {
384 requireRtpConnection().acceptCall();
385 } catch (final IllegalStateException e) {
386 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
387 }
388 }
389
390 private void putScreenInCallMode() {
391 putScreenInCallMode(requireRtpConnection().getMedia());
392 }
393
394 private void putScreenInCallMode(final Set<Media> media) {
395 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
396 if (Media.audioOnly(media)) {
397 final JingleRtpConnection rtpConnection =
398 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
399 final CallIntegration callIntegration =
400 rtpConnection == null ? null : rtpConnection.getCallIntegration();
401 if (callIntegration == null
402 || callIntegration.getSelectedAudioDevice()
403 == CallIntegration.AudioDevice.EARPIECE) {
404 acquireProximityWakeLock();
405 }
406 }
407 lockOrientation(media);
408 }
409
410 private void lockOrientation(final Set<Media> media) {
411 if (Media.audioOnly(media)) {
412 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
413 } else {
414 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
415 }
416 }
417
418 @SuppressLint("WakelockTimeout")
419 private void acquireProximityWakeLock() {
420 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
421 if (powerManager == null) {
422 Log.e(Config.LOGTAG, "power manager not available");
423 return;
424 }
425 if (isFinishing()) {
426 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
427 return;
428 }
429 if (this.mProximityWakeLock == null) {
430 this.mProximityWakeLock =
431 powerManager.newWakeLock(
432 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
433 }
434 if (!this.mProximityWakeLock.isHeld()) {
435 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
436 this.mProximityWakeLock.acquire();
437 }
438 }
439
440 private void releaseProximityWakeLock() {
441 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
442 Log.d(Config.LOGTAG, "releasing proximity wake lock");
443 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
444 this.mProximityWakeLock = null;
445 }
446 }
447
448 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
449 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
450 acquireProximityWakeLock();
451 } else {
452 releaseProximityWakeLock();
453 }
454 }
455
456 @Override
457 protected void refreshUiReal() {}
458
459 @Override
460 public void onNewIntent(final Intent intent) {
461 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
462 super.onNewIntent(intent);
463 setIntent(intent);
464 if (xmppConnectionService == null) {
465 Log.d(
466 Config.LOGTAG,
467 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
468 return;
469 }
470 final Account account = extractAccount(intent);
471 final String action = intent.getAction();
472 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
473 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
474 if (sessionId != null) {
475 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
476 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
477 return;
478 }
479 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
480 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
481 requestPermissionsAndAcceptCall();
482 resetIntent(intent.getExtras());
483 }
484 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
485 proposeJingleRtpSession(account, with, actionToMedia(action));
486 setWith(account.getRoster().getContact(with), null);
487 } else {
488 throw new IllegalStateException("received onNewIntent without sessionId");
489 }
490 }
491
492 @Override
493 void onBackendConnected() {
494 final Intent intent = getIntent();
495 final String action = intent.getAction();
496 final Account account = extractAccount(intent);
497 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
498 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
499 if (sessionId != null) {
500 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
501 return;
502 }
503 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
504 Log.d(Config.LOGTAG, "intent action was accept");
505 requestPermissionsAndAcceptCall();
506 resetIntent(intent.getExtras());
507 }
508 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
509 proposeJingleRtpSession(account, with, actionToMedia(action));
510 setWith(account.getRoster().getContact(with), null);
511 } else if (Intent.ACTION_VIEW.equals(action)) {
512 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
513 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
514 xmppConnectionService
515 .getJingleConnectionManager()
516 .getTerminalSessionState(with, proposedSessionId);
517 if (terminatedRtpSession != null) {
518 // termination (due to message error or 'busy' was faster than opening the activity
519 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
520 return;
521 }
522 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
523 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
524 final Set<Media> media = actionToMedia(lastAction);
525 final RtpEndUserState state =
526 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
527 if (state != null) {
528 Log.d(Config.LOGTAG, "restored last state from intent extra");
529 updateButtonConfiguration(state);
530 updateVerifiedShield(false);
531 updateStateDisplay(state);
532 updateIncomingCallScreen(state);
533 invalidateOptionsMenu();
534 }
535 setWith(account.getRoster().getContact(with), state);
536 if (xmppConnectionService
537 .getJingleConnectionManager()
538 .fireJingleRtpConnectionStateUpdates()) {
539 return;
540 }
541 if (END_CARD.contains(state)) {
542 return;
543 }
544 if (xmppConnectionService
545 .getJingleConnectionManager()
546 .hasMatchingProposal(account, with)) {
547 putScreenInCallMode(media);
548 return;
549 }
550 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
551 finish();
552 }
553 }
554
555 private void setWidth(final RtpEndUserState state) {
556 setWith(getWith(), state);
557 }
558
559 private void setWith(final Contact contact, final RtpEndUserState state) {
560 binding.with.setText(contact.getDisplayName());
561 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
562 .contains(state)) {
563 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
564 binding.withJid.setVisibility(View.VISIBLE);
565 } else {
566 binding.withJid.setVisibility(View.GONE);
567 }
568 }
569
570 private void proposeJingleRtpSession(
571 final Account account, final Jid with, final Set<Media> media) {
572 if (with.isBareJid()) {
573 xmppConnectionService
574 .getJingleConnectionManager()
575 .proposeJingleRtpSession(account, with, media);
576 } else {
577 throw new IllegalStateException(
578 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
579 }
580 putScreenInCallMode(media);
581 }
582
583 @Override
584 public void onRequestPermissionsResult(
585 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
586 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
587 final PermissionUtils.PermissionResult permissionResult =
588 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
589 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
590 if (requestCode == REQUEST_ACCEPT_CALL) {
591 acceptCall();
592 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
593 acceptContentAdd();
594 } else if (requestCode == REQUEST_ADD_CONTENT) {
595 switchToVideo();
596 }
597 } else {
598 @StringRes int res;
599 final String firstDenied =
600 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
601 if (firstDenied == null) {
602 return;
603 }
604 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
605 res = R.string.no_microphone_permission;
606 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
607 res = R.string.no_camera_permission;
608 } else {
609 throw new IllegalStateException("Invalid permission result request");
610 }
611 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
612 .show();
613 }
614 }
615
616 @Override
617 public void onStart() {
618 super.onStart();
619 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
620 this.binding.remoteVideo.setOnAspectRatioChanged(this);
621 }
622
623 @Override
624 public void onStop() {
625 mHandler.removeCallbacks(mTickExecutor);
626 binding.remoteVideo.release();
627 binding.remoteVideo.setOnAspectRatioChanged(null);
628 binding.localVideo.release();
629 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
630 final JingleRtpConnection jingleRtpConnection =
631 weakReference == null ? null : weakReference.get();
632 if (jingleRtpConnection != null) {
633 releaseVideoTracks(jingleRtpConnection);
634 }
635 releaseProximityWakeLock();
636 super.onStop();
637 }
638
639 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
640 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
641 if (remoteVideo.isPresent()) {
642 remoteVideo.get().removeSink(binding.remoteVideo);
643 }
644 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
645 if (localVideo.isPresent()) {
646 localVideo.get().removeSink(binding.localVideo);
647 }
648 }
649
650 @Override
651 public void onBackPressed() {
652 if (isConnected()) {
653 if (switchToPictureInPicture()) {
654 return;
655 }
656 } else {
657 endCall();
658 }
659 super.onBackPressed();
660 }
661
662 @Override
663 public void onUserLeaveHint() {
664 super.onUserLeaveHint();
665 if (switchToPictureInPicture()) {
666 return;
667 }
668 // TODO apparently this method is not getting called on Android 10 when using the task
669 // switcher
670 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
671 retractSessionProposal();
672 }
673 }
674
675 private boolean isConnected() {
676 final JingleRtpConnection connection =
677 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
678 final RtpEndUserState endUserState =
679 connection == null ? null : connection.getEndUserState();
680 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
681 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
682 }
683
684 private boolean switchToPictureInPicture() {
685 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
686 if (shouldBePictureInPicture()) {
687 startPictureInPicture();
688 return true;
689 }
690 }
691 return false;
692 }
693
694 @RequiresApi(api = Build.VERSION_CODES.O)
695 private void startPictureInPicture() {
696 try {
697 final Rational rational = this.binding.remoteVideo.getAspectRatio();
698 final Rational clippedRational = Rationals.clip(rational);
699 Log.d(
700 Config.LOGTAG,
701 "suggested rational " + rational + ". clipped to " + clippedRational);
702 enterPictureInPictureMode(
703 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
704 } catch (final IllegalStateException e) {
705 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
706 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
707 }
708 }
709
710 @Override
711 public void onAspectRatioChanged(final Rational rational) {
712 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
713 final Rational clippedRational = Rationals.clip(rational);
714 Log.d(
715 Config.LOGTAG,
716 "suggested rational after aspect ratio change "
717 + rational
718 + ". clipped to "
719 + clippedRational);
720 setPictureInPictureParams(
721 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
722 }
723 }
724
725 private boolean deviceSupportsPictureInPicture() {
726 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
727 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
728 } else {
729 return false;
730 }
731 }
732
733 private boolean shouldBePictureInPicture() {
734 try {
735 final JingleRtpConnection rtpConnection = requireRtpConnection();
736 return rtpConnection.getMedia().contains(Media.VIDEO)
737 && Arrays.asList(
738 RtpEndUserState.ACCEPTING_CALL,
739 RtpEndUserState.CONNECTING,
740 RtpEndUserState.CONNECTED)
741 .contains(rtpConnection.getEndUserState());
742 } catch (final IllegalStateException e) {
743 return false;
744 }
745 }
746
747 private boolean initializeActivityWithRunningRtpSession(
748 final Account account, Jid with, String sessionId) {
749 final WeakReference<JingleRtpConnection> reference =
750 xmppConnectionService
751 .getJingleConnectionManager()
752 .findJingleRtpConnection(account, with, sessionId);
753 if (reference == null || reference.get() == null) {
754 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
755 xmppConnectionService
756 .getJingleConnectionManager()
757 .getTerminalSessionState(with, sessionId);
758 if (terminatedRtpSession == null) {
759 throw new IllegalStateException(
760 "failed to initialize activity with running rtp session. session not found");
761 }
762 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
763 return true;
764 }
765 this.rtpConnectionReference = reference;
766 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
767 final boolean verified = requireRtpConnection().isVerified();
768 if (currentState == RtpEndUserState.ENDED) {
769 finish();
770 return true;
771 }
772 final Set<Media> media = getMedia();
773 final ContentAddition contentAddition = getPendingContentAddition();
774 if (currentState == RtpEndUserState.INCOMING_CALL) {
775 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
776 }
777 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
778 requireRtpConnection().getState())) {
779 putScreenInCallMode();
780 }
781 setWidth(currentState);
782 updateVideoViews(currentState);
783 updateStateDisplay(currentState, media, contentAddition);
784 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
785 updateButtonConfiguration(currentState, media, contentAddition);
786 updateIncomingCallScreen(currentState);
787 invalidateOptionsMenu();
788 return false;
789 }
790
791 private void initializeWithTerminatedSessionState(
792 final Account account,
793 final Jid with,
794 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
795 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
796 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
797 finish();
798 return;
799 }
800 final RtpEndUserState state = terminatedRtpSession.state;
801 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
802 updateButtonConfiguration(state);
803 updateStateDisplay(state);
804 updateIncomingCallScreen(state);
805 updateCallDuration();
806 updateVerifiedShield(false);
807 invalidateOptionsMenu();
808 setWith(account.getRoster().getContact(with), state);
809 }
810
811 private void reInitializeActivityWithRunningRtpSession(
812 final Account account, Jid with, String sessionId) {
813 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
814 resetIntent(account, with, sessionId);
815 }
816
817 private void resetIntent(final Account account, final Jid with, final String sessionId) {
818 final Intent intent = new Intent(Intent.ACTION_VIEW);
819 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
820 intent.putExtra(EXTRA_WITH, with.toEscapedString());
821 intent.putExtra(EXTRA_SESSION_ID, sessionId);
822 setIntent(intent);
823 }
824
825 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
826 surfaceViewRenderer.setVisibility(View.VISIBLE);
827 try {
828 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
829 } catch (final IllegalStateException ignored) {
830 // SurfaceViewRenderer was already initialized
831 } catch (final RuntimeException e) {
832 if (Throwables.getRootCause(e) instanceof GLException glException) {
833 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
834 }
835 }
836 surfaceViewRenderer.setEnableHardwareScaler(true);
837 }
838
839 private void updateStateDisplay(final RtpEndUserState state) {
840 updateStateDisplay(state, Collections.emptySet(), null);
841 }
842
843 private void updateStateDisplay(
844 final RtpEndUserState state,
845 final Set<Media> media,
846 final ContentAddition contentAddition) {
847 switch (state) {
848 case INCOMING_CALL -> {
849 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
850 if (media.contains(Media.VIDEO)) {
851 setTitle(R.string.rtp_state_incoming_video_call);
852 } else {
853 setTitle(R.string.rtp_state_incoming_call);
854 }
855 }
856 case INCOMING_CONTENT_ADD -> {
857 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
858 setTitle(R.string.rtp_state_content_add_video);
859 } else {
860 setTitle(R.string.rtp_state_content_add);
861 }
862 }
863 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
864 case CONNECTED -> setTitle(R.string.rtp_state_connected);
865 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
866 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
867 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
868 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
869 case RINGING -> setTitle(R.string.rtp_state_ringing);
870 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
871 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
872 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
873 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
874 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
875 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
876 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
877 case ENDED -> throw new IllegalStateException(
878 "Activity should have called finishAndReleaseWakeLock();");
879 default -> throw new IllegalStateException(
880 String.format("State %s has not been handled in UI", state));
881 }
882 }
883
884 private void updateVerifiedShield(final boolean verified) {
885 if (isPictureInPicture()) {
886 this.binding.verified.setVisibility(View.GONE);
887 return;
888 }
889 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
890 }
891
892 private void updateIncomingCallScreen(final RtpEndUserState state) {
893 updateIncomingCallScreen(state, null);
894 }
895
896 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
897 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
898 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
899 if (show) {
900 binding.contactPhoto.setVisibility(View.VISIBLE);
901 if (contact == null) {
902 AvatarWorkerTask.loadAvatar(
903 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
904 } else {
905 AvatarWorkerTask.loadAvatar(
906 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
907 }
908 } else {
909 binding.contactPhoto.setVisibility(View.GONE);
910 }
911 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
912 binding.usingAccount.setVisibility(View.VISIBLE);
913 binding.usingAccount.setText(
914 getString(
915 R.string.using_account,
916 account.getJid().asBareJid().toEscapedString()));
917 } else {
918 binding.usingAccount.setVisibility(View.GONE);
919 binding.contactPhoto.setVisibility(View.GONE);
920 }
921 }
922
923 private Set<Media> getMedia() {
924 return requireRtpConnection().getMedia();
925 }
926
927 public ContentAddition getPendingContentAddition() {
928 return requireRtpConnection().getPendingContentAddition();
929 }
930
931 private void updateButtonConfiguration(final RtpEndUserState state) {
932 updateButtonConfiguration(state, Collections.emptySet(), null);
933 }
934
935 @SuppressLint("RestrictedApi")
936 private void updateButtonConfiguration(
937 final RtpEndUserState state,
938 final Set<Media> media,
939 final ContentAddition contentAddition) {
940 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
941 this.binding.rejectCall.setVisibility(View.INVISIBLE);
942 this.binding.endCall.setVisibility(View.INVISIBLE);
943 this.binding.acceptCall.setVisibility(View.INVISIBLE);
944 } else if (state == RtpEndUserState.INCOMING_CALL) {
945 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
946 this.binding.rejectCall.setOnClickListener(this::rejectCall);
947 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
948 this.binding.rejectCall.setVisibility(View.VISIBLE);
949 this.binding.endCall.setVisibility(View.INVISIBLE);
950 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
951 this.binding.acceptCall.setOnClickListener(this::acceptCall);
952 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
953 this.binding.acceptCall.setVisibility(View.VISIBLE);
954 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
955 this.binding.rejectCall.setContentDescription(
956 getString(R.string.reject_switch_to_video));
957 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
958 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
959 this.binding.rejectCall.setVisibility(View.VISIBLE);
960 this.binding.endCall.setVisibility(View.INVISIBLE);
961 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
962 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
963 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
964 this.binding.acceptCall.setVisibility(View.VISIBLE);
965 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
966 .contains(state)) {
967 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
968 this.binding.rejectCall.setOnClickListener(this::exit);
969 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
970 this.binding.rejectCall.setVisibility(View.VISIBLE);
971 this.binding.endCall.setVisibility(View.INVISIBLE);
972 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
973 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
974 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
975 this.binding.acceptCall.setVisibility(View.VISIBLE);
976 } else if (asList(
977 RtpEndUserState.CONNECTIVITY_ERROR,
978 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
979 RtpEndUserState.APPLICATION_ERROR,
980 RtpEndUserState.RETRACTED,
981 RtpEndUserState.SECURITY_ERROR)
982 .contains(state)) {
983 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
984 this.binding.rejectCall.setOnClickListener(this::exit);
985 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
986 this.binding.rejectCall.setVisibility(View.VISIBLE);
987 this.binding.endCall.setVisibility(View.INVISIBLE);
988 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
989 this.binding.acceptCall.setOnClickListener(this::retry);
990 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
991 this.binding.acceptCall.setVisibility(View.VISIBLE);
992 } else {
993 this.binding.rejectCall.setVisibility(View.INVISIBLE);
994 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
995 this.binding.endCall.setOnClickListener(this::endCall);
996 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
997 this.binding.endCall.setVisibility(View.VISIBLE);
998 this.binding.acceptCall.setVisibility(View.INVISIBLE);
999 }
1000 updateInCallButtonConfiguration(state, media);
1001 }
1002
1003 private boolean isPictureInPicture() {
1004 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1005 return isInPictureInPictureMode();
1006 } else {
1007 return false;
1008 }
1009 }
1010
1011 private void updateInCallButtonConfiguration() {
1012 updateInCallButtonConfiguration(
1013 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1014 }
1015
1016 @SuppressLint("RestrictedApi")
1017 private void updateInCallButtonConfiguration(
1018 final RtpEndUserState state, final Set<Media> media) {
1019 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1020 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1021 if (media.contains(Media.VIDEO)) {
1022 final JingleRtpConnection rtpConnection = requireRtpConnection();
1023 updateInCallButtonConfigurationVideo(
1024 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1025 } else {
1026 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1027 updateInCallButtonConfigurationSpeaker(
1028 callIntegration.getSelectedAudioDevice(),
1029 callIntegration.getAudioDevices().size());
1030 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1031 }
1032 if (media.contains(Media.AUDIO)) {
1033 updateInCallButtonConfigurationMicrophone(
1034 requireRtpConnection().isMicrophoneEnabled());
1035 } else {
1036 this.binding.inCallActionLeft.setVisibility(View.GONE);
1037 }
1038 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1039 && !isPictureInPicture()
1040 && Media.audioOnly(media)) {
1041 final CallIntegration callIntegration;
1042 try {
1043 callIntegration = requireCallIntegration();
1044 } catch (final IllegalStateException e) {
1045 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1046 return;
1047 }
1048 updateInCallButtonConfigurationSpeaker(
1049 callIntegration.getSelectedAudioDevice(),
1050 callIntegration.getAudioDevices().size());
1051 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1052 } else {
1053 this.binding.inCallActionLeft.setVisibility(View.GONE);
1054 this.binding.inCallActionRight.setVisibility(View.GONE);
1055 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1056 }
1057 }
1058
1059 @SuppressLint("RestrictedApi")
1060 private void updateInCallButtonConfigurationSpeaker(
1061 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1062 switch (selectedAudioDevice) {
1063 case EARPIECE -> {
1064 this.binding.inCallActionRight.setImageResource(
1065 R.drawable.ic_volume_off_black_24dp);
1066 if (numberOfChoices >= 2) {
1067 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1068 } else {
1069 this.binding.inCallActionRight.setOnClickListener(null);
1070 this.binding.inCallActionRight.setClickable(false);
1071 }
1072 }
1073 case WIRED_HEADSET -> {
1074 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1075 this.binding.inCallActionRight.setOnClickListener(null);
1076 this.binding.inCallActionRight.setClickable(false);
1077 }
1078 case SPEAKER_PHONE -> {
1079 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1080 if (numberOfChoices >= 2) {
1081 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1082 } else {
1083 this.binding.inCallActionRight.setOnClickListener(null);
1084 this.binding.inCallActionRight.setClickable(false);
1085 }
1086 }
1087 case BLUETOOTH -> {
1088 this.binding.inCallActionRight.setImageResource(
1089 R.drawable.ic_bluetooth_audio_black_24dp);
1090 this.binding.inCallActionRight.setOnClickListener(null);
1091 this.binding.inCallActionRight.setClickable(false);
1092 }
1093 }
1094 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1095 }
1096
1097 @SuppressLint("RestrictedApi")
1098 private void updateInCallButtonConfigurationVideo(
1099 final boolean videoEnabled, final boolean isCameraSwitchable) {
1100 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1101 if (isCameraSwitchable) {
1102 this.binding.inCallActionFarRight.setImageResource(
1103 R.drawable.ic_flip_camera_android_black_24dp);
1104 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1105 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1106 } else {
1107 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1108 }
1109 if (videoEnabled) {
1110 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1111 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1112 } else {
1113 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1114 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1115 }
1116 }
1117
1118 private void switchCamera(final View view) {
1119 Futures.addCallback(
1120 requireRtpConnection().switchCamera(),
1121 new FutureCallback<>() {
1122 @Override
1123 public void onSuccess(@Nullable Boolean isFrontCamera) {
1124 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1125 }
1126
1127 @Override
1128 public void onFailure(@NonNull final Throwable throwable) {
1129 Log.d(
1130 Config.LOGTAG,
1131 "could not switch camera",
1132 Throwables.getRootCause(throwable));
1133 Toast.makeText(
1134 RtpSessionActivity.this,
1135 R.string.could_not_switch_camera,
1136 Toast.LENGTH_LONG)
1137 .show();
1138 }
1139 },
1140 MainThreadExecutor.getInstance());
1141 }
1142
1143 private void enableVideo(View view) {
1144 try {
1145 requireRtpConnection().setVideoEnabled(true);
1146 } catch (final IllegalStateException e) {
1147 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1148 return;
1149 }
1150 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1151 }
1152
1153 private void disableVideo(View view) {
1154 final JingleRtpConnection rtpConnection = requireRtpConnection();
1155 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1156 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1157 rtpConnection.retractContentAdd();
1158 return;
1159 }
1160 requireRtpConnection().setVideoEnabled(false);
1161 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1162 }
1163
1164 @SuppressLint("RestrictedApi")
1165 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1166 if (microphoneEnabled) {
1167 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1168 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1169 } else {
1170 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1171 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1172 }
1173 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1174 }
1175
1176 private void updateCallDuration() {
1177 final JingleRtpConnection connection =
1178 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1179 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1180 this.binding.duration.setVisibility(View.GONE);
1181 return;
1182 }
1183 if (connection.zeroDuration()) {
1184 this.binding.duration.setVisibility(View.GONE);
1185 } else {
1186 this.binding.duration.setText(
1187 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1188 this.binding.duration.setVisibility(View.VISIBLE);
1189 }
1190 }
1191
1192 private void updateVideoViews(final RtpEndUserState state) {
1193 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1194 binding.localVideo.setVisibility(View.GONE);
1195 binding.localVideo.release();
1196 binding.remoteVideoWrapper.setVisibility(View.GONE);
1197 binding.remoteVideo.release();
1198 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1199 if (isPictureInPicture()) {
1200 binding.appBarLayout.setVisibility(View.GONE);
1201 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1202 if (Arrays.asList(
1203 RtpEndUserState.APPLICATION_ERROR,
1204 RtpEndUserState.CONNECTIVITY_ERROR,
1205 RtpEndUserState.SECURITY_ERROR)
1206 .contains(state)) {
1207 binding.pipWarning.setVisibility(View.VISIBLE);
1208 binding.pipWaiting.setVisibility(View.GONE);
1209 } else {
1210 binding.pipWarning.setVisibility(View.GONE);
1211 binding.pipWaiting.setVisibility(View.GONE);
1212 }
1213 } else {
1214 binding.appBarLayout.setVisibility(View.VISIBLE);
1215 binding.pipPlaceholder.setVisibility(View.GONE);
1216 }
1217 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1218 return;
1219 }
1220 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1221 binding.localVideo.setVisibility(View.GONE);
1222 binding.remoteVideoWrapper.setVisibility(View.GONE);
1223 binding.appBarLayout.setVisibility(View.GONE);
1224 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1225 binding.pipWarning.setVisibility(View.GONE);
1226 binding.pipWaiting.setVisibility(View.VISIBLE);
1227 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1228 return;
1229 }
1230 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1231 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1232 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1233 // paint local view over remote view
1234 binding.localVideo.setZOrderMediaOverlay(true);
1235 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1236 addSink(localVideoTrack.get(), binding.localVideo);
1237 } else {
1238 binding.localVideo.setVisibility(View.GONE);
1239 }
1240 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1241 if (remoteVideoTrack.isPresent()) {
1242 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1243 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1244 binding.remoteVideo.setScalingType(
1245 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1246 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1247 if (state == RtpEndUserState.CONNECTED) {
1248 binding.appBarLayout.setVisibility(View.GONE);
1249 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1250 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1251 } else {
1252 binding.appBarLayout.setVisibility(View.VISIBLE);
1253 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1254 binding.remoteVideoWrapper.setVisibility(View.GONE);
1255 }
1256 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1257 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1258 } else {
1259 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1260 }
1261 } else {
1262 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1263 binding.remoteVideoWrapper.setVisibility(View.GONE);
1264 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1265 }
1266 }
1267
1268 private Optional<VideoTrack> getLocalVideoTrack() {
1269 final JingleRtpConnection connection =
1270 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1271 if (connection == null) {
1272 return Optional.absent();
1273 }
1274 return connection.getLocalVideoTrack();
1275 }
1276
1277 private Optional<VideoTrack> getRemoteVideoTrack() {
1278 final JingleRtpConnection connection =
1279 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1280 if (connection == null) {
1281 return Optional.absent();
1282 }
1283 return connection.getRemoteVideoTrack();
1284 }
1285
1286 private void disableMicrophone(View view) {
1287 final JingleRtpConnection rtpConnection = requireRtpConnection();
1288 if (rtpConnection.setMicrophoneEnabled(false)) {
1289 updateInCallButtonConfiguration();
1290 }
1291 }
1292
1293 private void enableMicrophone(View view) {
1294 final JingleRtpConnection rtpConnection = requireRtpConnection();
1295 if (rtpConnection.setMicrophoneEnabled(true)) {
1296 updateInCallButtonConfiguration();
1297 }
1298 }
1299
1300 private void switchToEarpiece(final View view) {
1301 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1302 acquireProximityWakeLock();
1303 }
1304
1305 private void switchToSpeaker(final View view) {
1306 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1307 releaseProximityWakeLock();
1308 }
1309
1310 private void retry(final View view) {
1311 final Intent intent = getIntent();
1312 final Account account = extractAccount(intent);
1313 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1314 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1315 final String action = intent.getAction();
1316 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1317 this.rtpConnectionReference = null;
1318 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1319 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1320 }
1321
1322 private void exit(final View view) {
1323 finish();
1324 }
1325
1326 private void recordVoiceMail(final View view) {
1327 final Intent intent = getIntent();
1328 final Account account = extractAccount(intent);
1329 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1330 final Conversation conversation =
1331 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1332 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1333 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1334 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1335 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1336 launchIntent.putExtra(
1337 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1338 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1339 startActivity(launchIntent);
1340 finish();
1341 }
1342
1343 private Contact getWith() {
1344 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1345 final Account account = id.account;
1346 return account.getRoster().getContact(id.with);
1347 }
1348
1349 private JingleRtpConnection requireRtpConnection() {
1350 final JingleRtpConnection connection =
1351 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1352 if (connection == null) {
1353 throw new IllegalStateException("No RTP connection found");
1354 }
1355 return connection;
1356 }
1357
1358 private CallIntegration requireCallIntegration() {
1359 return requireOngoingRtpSession().getCallIntegration();
1360 }
1361
1362 private OngoingRtpSession requireOngoingRtpSession() {
1363 final JingleRtpConnection connection =
1364 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1365 if (connection != null) {
1366 return connection;
1367 }
1368 final Intent currentIntent = getIntent();
1369 final String withExtra =
1370 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1371 final var account = extractAccount(currentIntent);
1372 if (withExtra == null) {
1373 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1374 }
1375 final var matching =
1376 xmppConnectionService
1377 .getJingleConnectionManager()
1378 .matchingProposal(account, Jid.of(withExtra));
1379 if (matching.isPresent()) {
1380 return matching.get();
1381 }
1382 throw new IllegalStateException("No matching session proposal");
1383 }
1384
1385 @Override
1386 public void onJingleRtpConnectionUpdate(
1387 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1388 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1389 if (END_CARD.contains(state)) {
1390 Log.d(Config.LOGTAG, "end card reached");
1391 releaseProximityWakeLock();
1392 runOnUiThread(
1393 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1394 }
1395 if (with.isBareJid()) {
1396 updateRtpSessionProposalState(account, with, state);
1397 return;
1398 }
1399 if (emptyReference(this.rtpConnectionReference)) {
1400 if (END_CARD.contains(state)) {
1401 Log.d(Config.LOGTAG, "not reinitializing session");
1402 return;
1403 }
1404 // this happens when going from proposed session to actual session
1405 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1406 return;
1407 }
1408 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1409 final boolean verified = requireRtpConnection().isVerified();
1410 final Set<Media> media = getMedia();
1411 lockOrientation(media);
1412 final ContentAddition contentAddition = getPendingContentAddition();
1413 final Contact contact = getWith();
1414 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1415 if (state == RtpEndUserState.ENDED) {
1416 finish();
1417 return;
1418 }
1419 runOnUiThread(
1420 () -> {
1421 updateStateDisplay(state, media, contentAddition);
1422 updateVerifiedShield(
1423 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1424 updateButtonConfiguration(state, media, contentAddition);
1425 updateVideoViews(state);
1426 updateIncomingCallScreen(state, contact);
1427 invalidateOptionsMenu();
1428 });
1429 if (END_CARD.contains(state)) {
1430 final JingleRtpConnection rtpConnection = requireRtpConnection();
1431 resetIntent(account, with, state, rtpConnection.getMedia());
1432 releaseVideoTracks(rtpConnection);
1433 this.rtpConnectionReference = null;
1434 }
1435 } else {
1436 Log.d(Config.LOGTAG, "received update for other rtp session");
1437 }
1438 }
1439
1440 @Override
1441 public void onAudioDeviceChanged(
1442 final CallIntegration.AudioDevice selectedAudioDevice,
1443 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1444 Log.d(
1445 Config.LOGTAG,
1446 "onAudioDeviceChanged in activity: selected:"
1447 + selectedAudioDevice
1448 + ", available:"
1449 + availableAudioDevices);
1450 try {
1451 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1452 final RtpEndUserState endUserState;
1453 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1454 endUserState = jingleRtpConnection.getEndUserState();
1455 } else {
1456 // for session proposals all end user states are functionally the same
1457 endUserState = RtpEndUserState.RINGING;
1458 }
1459 final Set<Media> media = ongoingRtpSession.getMedia();
1460 if (END_CARD.contains(endUserState)) {
1461 Log.d(
1462 Config.LOGTAG,
1463 "onAudioDeviceChanged() nothing to do because end card has been reached");
1464 } else {
1465 if (Media.audioOnly(media)
1466 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1467 final CallIntegration callIntegration = requireCallIntegration();
1468 updateInCallButtonConfigurationSpeaker(
1469 callIntegration.getSelectedAudioDevice(),
1470 callIntegration.getAudioDevices().size());
1471 }
1472 Log.d(
1473 Config.LOGTAG,
1474 "put proximity wake lock into proper state after device update");
1475 putProximityWakeLockInProperState(selectedAudioDevice);
1476 }
1477 } catch (final IllegalStateException e) {
1478 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1479 }
1480 }
1481
1482 private void updateRtpSessionProposalState(
1483 final Account account, final Jid with, final RtpEndUserState state) {
1484 final Intent currentIntent = getIntent();
1485 final String withExtra =
1486 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1487 if (withExtra == null) {
1488 return;
1489 }
1490 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1491 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1492 runOnUiThread(
1493 () -> {
1494 updateVerifiedShield(false);
1495 updateStateDisplay(state);
1496 updateButtonConfiguration(state, media, null);
1497 updateIncomingCallScreen(state);
1498 invalidateOptionsMenu();
1499 });
1500 resetIntent(account, with, state, media);
1501 }
1502 }
1503
1504 private void resetIntent(final Bundle extras) {
1505 final Intent intent = new Intent(Intent.ACTION_VIEW);
1506 intent.putExtras(extras);
1507 setIntent(intent);
1508 }
1509
1510 private void resetIntent(
1511 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1512 final Intent intent = new Intent(Intent.ACTION_VIEW);
1513 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1514 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1515 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1516 } else {
1517 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1518 }
1519 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1520 intent.putExtra(
1521 EXTRA_LAST_ACTION,
1522 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1523 setIntent(intent);
1524 }
1525
1526 private static boolean emptyReference(final WeakReference<?> weakReference) {
1527 return weakReference == null || weakReference.get() == null;
1528 }
1529}