1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.opengl.GLException;
16import android.os.Build;
17import android.os.Bundle;
18import android.os.Handler;
19import android.os.PowerManager;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Strings;
38import com.google.common.base.Throwables;
39import com.google.common.collect.ImmutableList;
40import com.google.common.collect.ImmutableSet;
41import com.google.common.util.concurrent.FutureCallback;
42import com.google.common.util.concurrent.Futures;
43
44import org.jetbrains.annotations.NotNull;
45import org.webrtc.RendererCommon;
46import org.webrtc.SurfaceViewRenderer;
47import org.webrtc.VideoTrack;
48
49import java.lang.ref.WeakReference;
50import java.util.Arrays;
51import java.util.Collections;
52import java.util.List;
53import java.util.Set;
54
55import eu.siacs.conversations.Config;
56import eu.siacs.conversations.R;
57import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
58import eu.siacs.conversations.entities.Account;
59import eu.siacs.conversations.entities.Contact;
60import eu.siacs.conversations.entities.Conversation;
61import eu.siacs.conversations.services.CallIntegration;
62import eu.siacs.conversations.services.CallIntegrationConnectionService;
63import eu.siacs.conversations.services.XmppConnectionService;
64import eu.siacs.conversations.ui.widget.DialpadView;
65import eu.siacs.conversations.ui.util.AvatarWorkerTask;
66import eu.siacs.conversations.ui.util.MainThreadExecutor;
67import eu.siacs.conversations.ui.util.Rationals;
68import eu.siacs.conversations.utils.PermissionUtils;
69import eu.siacs.conversations.utils.TimeFrameUtils;
70import eu.siacs.conversations.xmpp.Jid;
71import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
72import eu.siacs.conversations.xmpp.jingle.ContentAddition;
73import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
74import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
75import eu.siacs.conversations.xmpp.jingle.Media;
76import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession;
77import eu.siacs.conversations.xmpp.jingle.RtpCapability;
78import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
79
80import org.webrtc.RendererCommon;
81import org.webrtc.SurfaceViewRenderer;
82import org.webrtc.VideoTrack;
83
84import java.lang.ref.WeakReference;
85import java.util.Arrays;
86import java.util.Collections;
87import java.util.List;
88import java.util.Set;
89
90public class RtpSessionActivity extends XmppActivity
91 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
92 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
93
94 public static final String EXTRA_WITH = "with";
95 public static final String EXTRA_SESSION_ID = "session_id";
96 public static final String EXTRA_PROPOSED_SESSION_ID = "proposed_session_id";
97 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
98 public static final String EXTRA_LAST_ACTION = "last_action";
99 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
100 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
101 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
102
103 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
104
105 public static final List<RtpEndUserState> END_CARD =
106 Arrays.asList(
107 RtpEndUserState.APPLICATION_ERROR,
108 RtpEndUserState.SECURITY_ERROR,
109 RtpEndUserState.DECLINED_OR_BUSY,
110 RtpEndUserState.CONTACT_OFFLINE,
111 RtpEndUserState.CONNECTIVITY_ERROR,
112 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
113 RtpEndUserState.RETRACTED);
114 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
115 Arrays.asList(
116 RtpEndUserState.APPLICATION_ERROR,
117 RtpEndUserState.CONNECTIVITY_ERROR,
118 RtpEndUserState.SECURITY_ERROR);
119 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
120 Arrays.asList(
121 RtpEndUserState.CONNECTING,
122 RtpEndUserState.CONNECTED,
123 RtpEndUserState.RECONNECTING,
124 RtpEndUserState.INCOMING_CONTENT_ADD);
125 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
126 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
127 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
128 Arrays.asList(
129 RtpEndUserState.ACCEPTING_CALL,
130 RtpEndUserState.CONNECTING,
131 RtpEndUserState.RECONNECTING);
132 private static final List<RtpEndUserState> STATES_SHOWING_SPEAKER_CONFIGURATION =
133 new ImmutableList.Builder<RtpEndUserState>()
134 .add(RtpEndUserState.FINDING_DEVICE)
135 .add(RtpEndUserState.RINGING)
136 .add(RtpEndUserState.ACCEPTING_CALL)
137 .add(RtpEndUserState.CONNECTING)
138 .addAll(STATES_CONSIDERED_CONNECTED)
139 .build();
140 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
141 private static final int REQUEST_ACCEPT_CALL = 0x1111;
142 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
143 private static final int REQUEST_ADD_CONTENT = 0x1113;
144 private WeakReference<JingleRtpConnection> rtpConnectionReference;
145
146 private ActivityRtpSessionBinding binding;
147 private PowerManager.WakeLock mProximityWakeLock;
148
149 private final Handler mHandler = new Handler();
150 private final Runnable mTickExecutor =
151 new Runnable() {
152 @Override
153 public void run() {
154 updateCallDuration();
155 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
156 }
157 };
158
159 public static Set<Media> actionToMedia(final String action) {
160 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
161 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
162 } else if (ACTION_MAKE_VOICE_CALL.equals(action)) {
163 return ImmutableSet.of(Media.AUDIO);
164 } else {
165 Log.w(
166 Config.LOGTAG,
167 "actionToMedia can not get media set from unknown action " + action);
168 return Collections.emptySet();
169 }
170 }
171
172 private static void addSink(
173 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
174 try {
175 videoTrack.addSink(surfaceViewRenderer);
176 } catch (final IllegalStateException e) {
177 Log.e(
178 Config.LOGTAG,
179 "possible race condition on trying to display video track. ignoring",
180 e);
181 }
182 }
183
184 @Override
185 public void onCreate(Bundle savedInstanceState) {
186 super.onCreate(savedInstanceState);
187 getWindow()
188 .addFlags(
189 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
190 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
191 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
192 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
193 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
194 setSupportActionBar(binding.toolbar);
195
196 binding.dialpad.setClickConsumer(tag -> {
197 final JingleRtpConnection connection =
198 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
199 if (connection != null) connection.applyDtmfTone(tag);
200 });
201
202 if (savedInstanceState != null) {
203 boolean dialpadVisible = savedInstanceState.getBoolean("dialpad_visible");
204 binding.dialpad.setVisibility(dialpadVisible ? View.VISIBLE : View.GONE);
205 }
206
207 Activities.setStatusAndNavigationBarColors(this, binding.getRoot());
208 }
209
210 @Override
211 public boolean onCreateOptionsMenu(final Menu menu) {
212 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
213 final MenuItem help = menu.findItem(R.id.action_help);
214 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
215 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
216 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
217 help.setVisible(Config.HELP != null && isHelpButtonVisible());
218 gotoChat.setVisible(isSwitchToConversationVisible());
219 switchToVideo.setVisible(isSwitchToVideoVisible());
220 dialpad.setVisible(isAudioOnlyConversation());
221 return super.onCreateOptionsMenu(menu);
222 }
223
224 @Override
225 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
226 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
227 if (xmppConnectionService != null) {
228 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
229 return true;
230 }
231 }
232 }
233 return super.onKeyDown(keyCode, event);
234 }
235
236 private boolean isHelpButtonVisible() {
237 try {
238 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
239 } catch (IllegalStateException e) {
240 final Intent intent = getIntent();
241 final String state =
242 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
243 if (state != null) {
244 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
245 } else {
246 return false;
247 }
248 }
249 }
250
251 private boolean isSwitchToConversationVisible() {
252 final JingleRtpConnection connection =
253 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
254 return connection != null
255 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
256 }
257
258 private boolean isAudioOnlyConversation() {
259 final JingleRtpConnection connection =
260 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
261
262 return connection != null && !connection.getMedia().contains(Media.VIDEO);
263 }
264
265 private boolean isSwitchToVideoVisible() {
266 final JingleRtpConnection connection =
267 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
268 if (connection == null) {
269 return false;
270 }
271 return connection.isSwitchToVideoAvailable();
272 }
273
274 private void switchToConversation() {
275 final Contact contact = getWith();
276 final Conversation conversation =
277 xmppConnectionService.findOrCreateConversation(
278 contact.getAccount(), contact.getJid(), false, true);
279 switchToConversation(conversation);
280 }
281
282 private void toggleDialpadVisibility() {
283 if (binding.dialpad.getVisibility() == View.VISIBLE) {
284 binding.dialpad.setVisibility(View.GONE);
285 }
286 else {
287 binding.dialpad.setVisibility(View.VISIBLE);
288 }
289 }
290
291 public boolean onOptionsItemSelected(final MenuItem item) {
292 final var itemItem = item.getItemId();
293 if (itemItem == R.id.action_help) {
294 launchHelpInBrowser();
295 return true;
296 } else if (itemItem == R.id.action_goto_chat) {
297 switchToConversation();
298 return true;
299 } else if (itemItem == R.id.action_dialpad) {
300 toggleDialpadVisibility();
301 return true;
302 } else if (itemItem == R.id.action_switch_to_video) {
303 requestPermissionAndSwitchToVideo();
304 return true;
305 } else {
306 return super.onOptionsItemSelected(item);
307 }
308 }
309
310 private void launchHelpInBrowser() {
311 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
312 try {
313 startActivity(intent);
314 } catch (final ActivityNotFoundException e) {
315 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
316 .show();
317 }
318 }
319
320 private void endCall(View view) {
321 endCall();
322 }
323
324 private void endCall() {
325 if (this.rtpConnectionReference == null) {
326 retractSessionProposal();
327 finish();
328 } else {
329 try {
330 requireRtpConnection().endCall();
331 } catch (final IllegalStateException e) {
332 // No call, already done
333 finish();
334 }
335 }
336 }
337
338 private void retractSessionProposal() {
339 final Intent intent = getIntent();
340 final String action = intent.getAction();
341 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
342 final Account account = extractAccount(intent);
343 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
344 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
345 if (!Intent.ACTION_VIEW.equals(action)
346 || state == null
347 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
348 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
349 resetIntent(account, with, RtpEndUserState.RETRACTED, media);
350 }
351 xmppConnectionService
352 .getJingleConnectionManager()
353 .retractSessionProposal(account, with.asBareJid());
354 }
355
356 private void rejectCall(View view) {
357 requireRtpConnection().rejectCall();
358 finish();
359 }
360
361 private void acceptCall(View view) {
362 requestPermissionsAndAcceptCall();
363 }
364
365 private void acceptContentAdd() {
366 try {
367 final ContentAddition pendingContentAddition =
368 requireRtpConnection().getPendingContentAddition();
369 if (pendingContentAddition == null) {
370 Log.d(Config.LOGTAG, "content offer was gone after granting permission");
371 return;
372 }
373 requireRtpConnection().acceptContentAdd(pendingContentAddition.summary);
374 } catch (final IllegalStateException e) {
375 Toast.makeText(this, Strings.nullToEmpty(e.getMessage()), Toast.LENGTH_SHORT).show();
376 }
377 }
378
379 private void requestPermissionAndSwitchToVideo() {
380 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
381 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
382 switchToVideo();
383 }
384 }
385
386 private void switchToVideo() {
387 try {
388 requireRtpConnection().addMedia(Media.VIDEO);
389 } catch (final IllegalStateException e) {
390 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
391 }
392 }
393
394 private void acceptContentAdd(final ContentAddition contentAddition) {
395 if (contentAddition == null
396 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
397 Log.d(Config.LOGTAG, "ignore press on content-accept button");
398 return;
399 }
400 requestPermissionAndAcceptContentAdd(contentAddition);
401 }
402
403 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
404 final List<String> permissions = permissions(contentAddition.media());
405 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
406 try {
407 requireRtpConnection().acceptContentAdd(contentAddition.summary);
408 } catch (final IllegalStateException e) {
409 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
410 }
411 }
412 }
413
414 private void rejectContentAdd(final View view) {
415 requireRtpConnection().rejectContentAdd();
416 }
417
418 private void requestPermissionsAndAcceptCall() {
419 final List<String> permissions = permissions(getMedia());
420 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
421 putScreenInCallMode();
422 acceptCall();
423 }
424 }
425
426 private List<String> permissions(final Set<Media> media) {
427 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
428 if (media.contains(Media.VIDEO)) {
429 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
430 } else {
431 permissions.add(Manifest.permission.RECORD_AUDIO);
432 }
433 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
434 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
435 }
436 return permissions.build();
437 }
438
439 private void acceptCall() {
440 try {
441 requireRtpConnection().acceptCall();
442 } catch (final IllegalStateException e) {
443 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
444 }
445 }
446
447 private void putScreenInCallMode() {
448 putScreenInCallMode(requireRtpConnection().getMedia());
449 }
450
451 private void putScreenInCallMode(final Set<Media> media) {
452 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
453 if (Media.audioOnly(media)) {
454 final JingleRtpConnection rtpConnection =
455 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
456 final CallIntegration callIntegration =
457 rtpConnection == null ? null : rtpConnection.getCallIntegration();
458 if (callIntegration == null
459 || callIntegration.getSelectedAudioDevice()
460 == CallIntegration.AudioDevice.EARPIECE) {
461 acquireProximityWakeLock();
462 }
463 }
464 lockOrientation(media);
465 }
466
467 private void lockOrientation(final Set<Media> media) {
468 if (Media.audioOnly(media)) {
469 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
470 } else {
471 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
472 }
473 }
474
475 @SuppressLint("WakelockTimeout")
476 private void acquireProximityWakeLock() {
477 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
478 if (powerManager == null) {
479 Log.e(Config.LOGTAG, "power manager not available");
480 return;
481 }
482 if (isFinishing()) {
483 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
484 return;
485 }
486 if (this.mProximityWakeLock == null) {
487 this.mProximityWakeLock =
488 powerManager.newWakeLock(
489 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
490 }
491 if (!this.mProximityWakeLock.isHeld()) {
492 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
493 this.mProximityWakeLock.acquire();
494 }
495 }
496
497 private void releaseProximityWakeLock() {
498 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
499 Log.d(Config.LOGTAG, "releasing proximity wake lock");
500 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
501 this.mProximityWakeLock = null;
502 }
503 }
504
505 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
506 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
507 acquireProximityWakeLock();
508 } else {
509 releaseProximityWakeLock();
510 }
511 }
512
513 @Override
514 protected void refreshUiReal() {}
515
516 @Override
517 public void onNewIntent(final Intent intent) {
518 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
519 super.onNewIntent(intent);
520 if (intent == null) {
521 return;
522 }
523 setIntent(intent);
524 if (xmppConnectionService == null) {
525 Log.d(
526 Config.LOGTAG,
527 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
528 return;
529 }
530 initializeWithIntent(Event.ON_NEW_INTENT, intent);
531 }
532
533 @Override
534 protected void onBackendConnected() {
535 final var intent = getIntent();
536 if (intent == null) {
537 return;
538 }
539 initializeWithIntent(Event.ON_BACKEND_CONNECTED, intent);
540 }
541
542 private void initializeWithIntent(final Event event, @NonNull final Intent intent) {
543 final String action = intent.getAction();
544 Log.d(Config.LOGTAG, "initializeWithIntent(" + event + "," + action + ")");
545 final Account account = extractAccount(intent);
546 final var extraWith = intent.getStringExtra(EXTRA_WITH);
547 final Jid with = Strings.isNullOrEmpty(extraWith) ? null : Jid.ofEscaped(extraWith);
548 if (with == null || account == null) {
549 Log.e(Config.LOGTAG, "intent is missing extras (account or with)");
550 return;
551 }
552 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
553 if (sessionId != null) {
554 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
555 return;
556 }
557 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
558 Log.d(Config.LOGTAG, "intent action was accept");
559 requestPermissionsAndAcceptCall();
560 resetIntent(intent.getExtras());
561 }
562 } else if (Intent.ACTION_VIEW.equals(action)) {
563 final String proposedSessionId = intent.getStringExtra(EXTRA_PROPOSED_SESSION_ID);
564 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
565 xmppConnectionService
566 .getJingleConnectionManager()
567 .getTerminalSessionState(with, proposedSessionId);
568 if (terminatedRtpSession != null) {
569 // termination (due to message error or 'busy' was faster than opening the activity
570 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
571 return;
572 }
573 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
574 final RtpEndUserState state =
575 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
576 if (state != null) {
577 Log.d(Config.LOGTAG, "restored last state from intent extra");
578 updateButtonConfiguration(state);
579 updateVerifiedShield(false);
580 updateStateDisplay(state);
581 updateIncomingCallScreen(state);
582 invalidateOptionsMenu();
583 }
584 setWith(account.getRoster().getContact(with), state);
585 if (xmppConnectionService
586 .getJingleConnectionManager()
587 .fireJingleRtpConnectionStateUpdates()) {
588 return;
589 }
590 if (END_CARD.contains(state)) {
591 return;
592 }
593 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
594 final Set<Media> media = actionToMedia(lastAction);
595 if (xmppConnectionService
596 .getJingleConnectionManager()
597 .hasMatchingProposal(account, with)) {
598 putScreenInCallMode(media);
599 return;
600 }
601 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
602 finish();
603 }
604 }
605
606 private void setWith(final RtpEndUserState state) {
607 setWith(getWith(), state);
608 }
609
610 private void setWith(final Contact contact, final RtpEndUserState state) {
611 binding.with.setText(contact.getDisplayName());
612 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
613 .contains(state)) {
614 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
615 binding.withJid.setVisibility(View.VISIBLE);
616 } else {
617 binding.withJid.setVisibility(View.GONE);
618 }
619 }
620
621 @Override
622 public void onRequestPermissionsResult(
623 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
624 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
625 final PermissionUtils.PermissionResult permissionResult =
626 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
627 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
628 if (requestCode == REQUEST_ACCEPT_CALL) {
629 acceptCall();
630 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
631 acceptContentAdd();
632 } else if (requestCode == REQUEST_ADD_CONTENT) {
633 switchToVideo();
634 }
635 } else {
636 @StringRes int res;
637 final String firstDenied =
638 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
639 if (firstDenied == null) {
640 return;
641 }
642 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
643 res = R.string.no_microphone_permission;
644 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
645 res = R.string.no_camera_permission;
646 } else {
647 throw new IllegalStateException("Invalid permission result request");
648 }
649 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
650 .show();
651 }
652 }
653
654 @Override
655 public void onStart() {
656 super.onStart();
657 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
658 this.binding.remoteVideo.setOnAspectRatioChanged(this);
659 }
660
661 @Override
662 public void onStop() {
663 mHandler.removeCallbacks(mTickExecutor);
664 binding.remoteVideo.release();
665 binding.remoteVideo.setOnAspectRatioChanged(null);
666 binding.localVideo.release();
667 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
668 final JingleRtpConnection jingleRtpConnection =
669 weakReference == null ? null : weakReference.get();
670 if (jingleRtpConnection != null) {
671 releaseVideoTracks(jingleRtpConnection);
672 }
673 releaseProximityWakeLock();
674 super.onStop();
675 }
676
677 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
678 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
679 if (remoteVideo.isPresent()) {
680 remoteVideo.get().removeSink(binding.remoteVideo);
681 }
682 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
683 if (localVideo.isPresent()) {
684 localVideo.get().removeSink(binding.localVideo);
685 }
686 }
687
688 @Override
689 public void onBackPressed() {
690 if (isConnected()) {
691 if (switchToPictureInPicture()) {
692 return;
693 }
694 } else {
695 endCall();
696 }
697 super.onBackPressed();
698 }
699
700 @Override
701 public void onUserLeaveHint() {
702 super.onUserLeaveHint();
703 if (switchToPictureInPicture()) {
704 return;
705 }
706 // TODO apparently this method is not getting called on Android 10 when using the task
707 // switcher
708 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
709 retractSessionProposal();
710 }
711 }
712
713 private boolean isConnected() {
714 final JingleRtpConnection connection =
715 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
716 final RtpEndUserState endUserState =
717 connection == null ? null : connection.getEndUserState();
718 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
719 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
720 }
721
722 private boolean switchToPictureInPicture() {
723 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
724 if (shouldBePictureInPicture()) {
725 startPictureInPicture();
726 return true;
727 }
728 }
729 return false;
730 }
731
732 @RequiresApi(api = Build.VERSION_CODES.O)
733 private void startPictureInPicture() {
734 try {
735 final Rational rational = this.binding.remoteVideo.getAspectRatio();
736 final Rational clippedRational = Rationals.clip(rational);
737 Log.d(
738 Config.LOGTAG,
739 "suggested rational " + rational + ". clipped to " + clippedRational);
740 enterPictureInPictureMode(
741 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
742 } catch (final IllegalStateException e) {
743 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
744 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
745 }
746 }
747
748 @Override
749 public void onAspectRatioChanged(final Rational rational) {
750 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
751 final Rational clippedRational = Rationals.clip(rational);
752 Log.d(
753 Config.LOGTAG,
754 "suggested rational after aspect ratio change "
755 + rational
756 + ". clipped to "
757 + clippedRational);
758 setPictureInPictureParams(
759 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
760 }
761 }
762
763 private boolean deviceSupportsPictureInPicture() {
764 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
765 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
766 } else {
767 return false;
768 }
769 }
770
771 private boolean shouldBePictureInPicture() {
772 try {
773 final JingleRtpConnection rtpConnection = requireRtpConnection();
774 return rtpConnection.getMedia().contains(Media.VIDEO)
775 && Arrays.asList(
776 RtpEndUserState.ACCEPTING_CALL,
777 RtpEndUserState.CONNECTING,
778 RtpEndUserState.CONNECTED)
779 .contains(rtpConnection.getEndUserState());
780 } catch (final IllegalStateException e) {
781 return false;
782 }
783 }
784
785 private boolean initializeActivityWithRunningRtpSession(
786 final Account account, Jid with, String sessionId) {
787 final WeakReference<JingleRtpConnection> reference =
788 xmppConnectionService
789 .getJingleConnectionManager()
790 .findJingleRtpConnection(account, with, sessionId);
791 if (reference == null || reference.get() == null) {
792 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
793 xmppConnectionService
794 .getJingleConnectionManager()
795 .getTerminalSessionState(with, sessionId);
796 if (terminatedRtpSession == null) {
797 Log.e(Config.LOGTAG, "failed to initialize activity with running rtp session. session not found");
798 finish();
799 return true;
800 }
801 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
802 return true;
803 }
804 this.rtpConnectionReference = reference;
805 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
806 final boolean verified = requireRtpConnection().isVerified();
807 if (currentState == RtpEndUserState.ENDED) {
808 finish();
809 return true;
810 }
811 final Set<Media> media = getMedia();
812 final ContentAddition contentAddition = getPendingContentAddition();
813 if (currentState == RtpEndUserState.INCOMING_CALL) {
814 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
815 }
816 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
817 requireRtpConnection().getState())) {
818 putScreenInCallMode();
819 }
820 setWith(currentState);
821 updateVideoViews(currentState);
822 updateStateDisplay(currentState, media, contentAddition);
823 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
824 updateButtonConfiguration(currentState, media, contentAddition);
825 updateIncomingCallScreen(currentState);
826 invalidateOptionsMenu();
827 return false;
828 }
829
830 private void initializeWithTerminatedSessionState(
831 final Account account,
832 final Jid with,
833 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
834 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
835 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
836 finish();
837 return;
838 }
839 final RtpEndUserState state = terminatedRtpSession.state;
840 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
841 updateButtonConfiguration(state);
842 updateStateDisplay(state);
843 updateIncomingCallScreen(state);
844 updateCallDuration();
845 updateVerifiedShield(false);
846 invalidateOptionsMenu();
847 setWith(account.getRoster().getContact(with), state);
848 }
849
850 private void reInitializeActivityWithRunningRtpSession(
851 final Account account, Jid with, String sessionId) {
852 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
853 resetIntent(account, with, sessionId);
854 }
855
856 private void resetIntent(final Account account, final Jid with, final String sessionId) {
857 final Intent intent = new Intent(Intent.ACTION_VIEW);
858 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
859 intent.putExtra(EXTRA_WITH, with.toEscapedString());
860 intent.putExtra(EXTRA_SESSION_ID, sessionId);
861 setIntent(intent);
862 }
863
864 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
865 surfaceViewRenderer.setVisibility(View.VISIBLE);
866 try {
867 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
868 } catch (final IllegalStateException ignored) {
869 // SurfaceViewRenderer was already initialized
870 } catch (final RuntimeException e) {
871 if (Throwables.getRootCause(e) instanceof GLException glException) {
872 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
873 }
874 }
875 surfaceViewRenderer.setEnableHardwareScaler(true);
876 }
877
878 private void updateStateDisplay(final RtpEndUserState state) {
879 updateStateDisplay(state, Collections.emptySet(), null);
880 }
881
882 private void updateStateDisplay(
883 final RtpEndUserState state,
884 final Set<Media> media,
885 final ContentAddition contentAddition) {
886 switch (state) {
887 case INCOMING_CALL -> {
888 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
889 if (media.contains(Media.VIDEO)) {
890 setTitle(R.string.rtp_state_incoming_video_call);
891 } else {
892 setTitle(R.string.rtp_state_incoming_call);
893 }
894 }
895 case INCOMING_CONTENT_ADD -> {
896 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
897 setTitle(R.string.rtp_state_content_add_video);
898 } else {
899 setTitle(R.string.rtp_state_content_add);
900 }
901 }
902 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
903 case CONNECTED -> setTitle(R.string.rtp_state_connected);
904 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
905 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
906 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
907 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
908 case RINGING -> setTitle(R.string.rtp_state_ringing);
909 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
910 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
911 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
912 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
913 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
914 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
915 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
916 case ENDED -> throw new IllegalStateException(
917 "Activity should have called finishAndReleaseWakeLock();");
918 default -> throw new IllegalStateException(
919 String.format("State %s has not been handled in UI", state));
920 }
921 }
922
923 private void updateVerifiedShield(final boolean verified) {
924 if (isPictureInPicture()) {
925 this.binding.verified.setVisibility(View.GONE);
926 return;
927 }
928 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
929 }
930
931 private void updateIncomingCallScreen(final RtpEndUserState state) {
932 updateIncomingCallScreen(state, null);
933 }
934
935 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
936 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
937 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
938 if (show) {
939 binding.contactPhoto.setVisibility(View.VISIBLE);
940 if (contact == null) {
941 AvatarWorkerTask.loadAvatar(
942 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
943 } else {
944 AvatarWorkerTask.loadAvatar(
945 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
946 }
947 } else {
948 binding.contactPhoto.setVisibility(View.GONE);
949 }
950 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
951 binding.usingAccount.setVisibility(View.VISIBLE);
952 binding.usingAccount.setText(
953 getString(
954 R.string.using_account,
955 account.getJid().asBareJid().toEscapedString()));
956 } else {
957 binding.usingAccount.setVisibility(View.GONE);
958 binding.contactPhoto.setVisibility(View.GONE);
959 }
960 }
961
962 private Set<Media> getMedia() {
963 return requireRtpConnection().getMedia();
964 }
965
966 public ContentAddition getPendingContentAddition() {
967 return requireRtpConnection().getPendingContentAddition();
968 }
969
970 private void updateButtonConfiguration(final RtpEndUserState state) {
971 updateButtonConfiguration(state, Collections.emptySet(), null);
972 }
973
974 @SuppressLint("RestrictedApi")
975 private void updateButtonConfiguration(
976 final RtpEndUserState state,
977 final Set<Media> media,
978 final ContentAddition contentAddition) {
979 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
980 this.binding.rejectCall.setVisibility(View.INVISIBLE);
981 this.binding.endCall.setVisibility(View.INVISIBLE);
982 this.binding.acceptCall.setVisibility(View.INVISIBLE);
983 } else if (state == RtpEndUserState.INCOMING_CALL) {
984 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
985 this.binding.rejectCall.setOnClickListener(this::rejectCall);
986 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp);
987 this.binding.rejectCall.setVisibility(View.VISIBLE);
988 this.binding.endCall.setVisibility(View.INVISIBLE);
989 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
990 this.binding.acceptCall.setOnClickListener(this::acceptCall);
991 this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp);
992 this.binding.acceptCall.setVisibility(View.VISIBLE);
993 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
994 this.binding.rejectCall.setContentDescription(
995 getString(R.string.reject_switch_to_video));
996 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
997 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
998 this.binding.rejectCall.setVisibility(View.VISIBLE);
999 this.binding.endCall.setVisibility(View.INVISIBLE);
1000 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
1001 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
1002 this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp);
1003 this.binding.acceptCall.setVisibility(View.VISIBLE);
1004 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
1005 .contains(state)) {
1006 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1007 this.binding.rejectCall.setOnClickListener(this::exit);
1008 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1009 this.binding.rejectCall.setVisibility(View.VISIBLE);
1010 this.binding.endCall.setVisibility(View.INVISIBLE);
1011 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
1012 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
1013 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp);
1014 this.binding.acceptCall.setVisibility(View.VISIBLE);
1015 } else if (asList(
1016 RtpEndUserState.CONNECTIVITY_ERROR,
1017 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
1018 RtpEndUserState.APPLICATION_ERROR,
1019 RtpEndUserState.RETRACTED,
1020 RtpEndUserState.SECURITY_ERROR)
1021 .contains(state)) {
1022 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1023 this.binding.rejectCall.setOnClickListener(this::exit);
1024 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp);
1025 this.binding.rejectCall.setVisibility(View.VISIBLE);
1026 this.binding.endCall.setVisibility(View.INVISIBLE);
1027 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1028 this.binding.acceptCall.setOnClickListener(this::retry);
1029 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp);
1030 this.binding.acceptCall.setVisibility(View.VISIBLE);
1031 } else {
1032 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1033 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1034 this.binding.endCall.setOnClickListener(this::endCall);
1035 this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp);
1036 this.binding.endCall.setVisibility(View.VISIBLE);
1037 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1038 }
1039 updateInCallButtonConfiguration(state, media);
1040 }
1041
1042 private boolean isPictureInPicture() {
1043 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1044 return isInPictureInPictureMode();
1045 } else {
1046 return false;
1047 }
1048 }
1049
1050 private void updateInCallButtonConfiguration() {
1051 updateInCallButtonConfiguration(
1052 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1053 }
1054
1055 @SuppressLint("RestrictedApi")
1056 private void updateInCallButtonConfiguration(
1057 final RtpEndUserState state, final Set<Media> media) {
1058 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1059 Preconditions.checkArgument(!media.isEmpty(), "Media must not be empty");
1060 if (media.contains(Media.VIDEO)) {
1061 final JingleRtpConnection rtpConnection = requireRtpConnection();
1062 updateInCallButtonConfigurationVideo(
1063 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1064 } else {
1065 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1066 updateInCallButtonConfigurationSpeaker(
1067 callIntegration.getSelectedAudioDevice(),
1068 callIntegration.getAudioDevices().size());
1069 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1070 }
1071 if (media.contains(Media.AUDIO)) {
1072 updateInCallButtonConfigurationMicrophone(
1073 requireRtpConnection().isMicrophoneEnabled());
1074 } else {
1075 this.binding.inCallActionLeft.setVisibility(View.GONE);
1076 }
1077 } else if (STATES_SHOWING_SPEAKER_CONFIGURATION.contains(state)
1078 && !isPictureInPicture()
1079 && Media.audioOnly(media)) {
1080 final CallIntegration callIntegration;
1081 try {
1082 callIntegration = requireCallIntegration();
1083 } catch (final IllegalStateException e) {
1084 Log.e(Config.LOGTAG, "can not update InCallButtonConfiguration in state " + state);
1085 return;
1086 }
1087 updateInCallButtonConfigurationSpeaker(
1088 callIntegration.getSelectedAudioDevice(),
1089 callIntegration.getAudioDevices().size());
1090 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1091 } else {
1092 this.binding.inCallActionLeft.setVisibility(View.GONE);
1093 this.binding.inCallActionRight.setVisibility(View.GONE);
1094 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1095 }
1096 }
1097
1098 @SuppressLint("RestrictedApi")
1099 private void updateInCallButtonConfigurationSpeaker(
1100 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1101 switch (selectedAudioDevice) {
1102 case EARPIECE -> {
1103 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_24dp);
1104 if (numberOfChoices >= 2) {
1105 this.binding.inCallActionRight.setContentDescription(
1106 getString(R.string.call_is_using_earpiece_tap_to_switch_to_speaker));
1107 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1108 } else {
1109 this.binding.inCallActionRight.setContentDescription(
1110 getString(R.string.call_is_using_earpiece));
1111 this.binding.inCallActionRight.setOnClickListener(null);
1112 this.binding.inCallActionRight.setClickable(false);
1113 }
1114 }
1115 case WIRED_HEADSET -> {
1116 this.binding.inCallActionRight.setContentDescription(
1117 getString(R.string.call_is_using_wired_headset));
1118 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_mic_24dp);
1119 this.binding.inCallActionRight.setOnClickListener(null);
1120 this.binding.inCallActionRight.setClickable(false);
1121 }
1122 case SPEAKER_PHONE -> {
1123 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp);
1124 if (numberOfChoices >= 2) {
1125 this.binding.inCallActionRight.setContentDescription(
1126 getString(R.string.call_is_using_speaker_tap_to_switch_to_earpiece));
1127 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1128 } else {
1129 this.binding.inCallActionRight.setContentDescription(
1130 getString(R.string.call_is_using_speaker));
1131 this.binding.inCallActionRight.setOnClickListener(null);
1132 this.binding.inCallActionRight.setClickable(false);
1133 }
1134 }
1135 case BLUETOOTH -> {
1136 this.binding.inCallActionRight.setContentDescription(
1137 getString(R.string.call_is_using_bluetooth));
1138 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_24dp);
1139 this.binding.inCallActionRight.setOnClickListener(null);
1140 this.binding.inCallActionRight.setClickable(false);
1141 }
1142 }
1143 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1144 }
1145
1146 @SuppressLint("RestrictedApi")
1147 private void updateInCallButtonConfigurationVideo(
1148 final boolean videoEnabled, final boolean isCameraSwitchable) {
1149 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1150 if (isCameraSwitchable) {
1151 this.binding.inCallActionFarRight.setImageResource(
1152 R.drawable.ic_flip_camera_android_24dp);
1153 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1154 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1155 this.binding.inCallActionFarRight.setContentDescription(
1156 getString(R.string.flip_camera));
1157 } else {
1158 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1159 }
1160 if (videoEnabled) {
1161 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp);
1162 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1163 this.binding.inCallActionRight.setContentDescription(
1164 getString(R.string.video_is_enabled_tap_to_disable));
1165 } else {
1166 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp);
1167 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1168 this.binding.inCallActionRight.setContentDescription(
1169 getString(R.string.video_is_disabled_tap_to_enable));
1170 }
1171 }
1172
1173 private void switchCamera(final View view) {
1174 Futures.addCallback(
1175 requireRtpConnection().switchCamera(),
1176 new FutureCallback<>() {
1177 @Override
1178 public void onSuccess(@Nullable Boolean isFrontCamera) {
1179 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1180 }
1181
1182 @Override
1183 public void onFailure(@NonNull final Throwable throwable) {
1184 Log.d(
1185 Config.LOGTAG,
1186 "could not switch camera",
1187 Throwables.getRootCause(throwable));
1188 Toast.makeText(
1189 RtpSessionActivity.this,
1190 R.string.could_not_switch_camera,
1191 Toast.LENGTH_LONG)
1192 .show();
1193 }
1194 },
1195 MainThreadExecutor.getInstance());
1196 }
1197
1198 private void enableVideo(final View view) {
1199 try {
1200 requireRtpConnection().setVideoEnabled(true);
1201 } catch (final IllegalStateException e) {
1202 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1203 return;
1204 }
1205 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1206 }
1207
1208 private void disableVideo(final View view) {
1209 final JingleRtpConnection rtpConnection = requireRtpConnection();
1210 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1211 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1212 rtpConnection.retractContentAdd();
1213 return;
1214 }
1215 try {
1216 requireRtpConnection().setVideoEnabled(false);
1217 } catch (final IllegalStateException e) {
1218 Toast.makeText(this, R.string.could_not_disable_video, Toast.LENGTH_SHORT).show();
1219 return;
1220 }
1221 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1222 }
1223
1224 @SuppressLint("RestrictedApi")
1225 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1226 if (microphoneEnabled) {
1227 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp);
1228 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1229 } else {
1230 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp);
1231 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1232 }
1233 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1234 }
1235
1236 private void updateCallDuration() {
1237 final JingleRtpConnection connection =
1238 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1239 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1240 this.binding.duration.setVisibility(View.GONE);
1241 return;
1242 }
1243 if (connection.zeroDuration()) {
1244 this.binding.duration.setVisibility(View.GONE);
1245 } else {
1246 this.binding.duration.setText(
1247 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1248 this.binding.duration.setVisibility(View.VISIBLE);
1249 }
1250 }
1251
1252 private void updateVideoViews(final RtpEndUserState state) {
1253 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1254 binding.localVideo.setVisibility(View.GONE);
1255 binding.localVideo.release();
1256 binding.remoteVideoWrapper.setVisibility(View.GONE);
1257 binding.remoteVideo.release();
1258 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1259 if (isPictureInPicture()) {
1260 binding.appBarLayout.setVisibility(View.GONE);
1261 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1262 if (Arrays.asList(
1263 RtpEndUserState.APPLICATION_ERROR,
1264 RtpEndUserState.CONNECTIVITY_ERROR,
1265 RtpEndUserState.SECURITY_ERROR)
1266 .contains(state)) {
1267 binding.pipWarning.setVisibility(View.VISIBLE);
1268 binding.pipWaiting.setVisibility(View.GONE);
1269 } else {
1270 binding.pipWarning.setVisibility(View.GONE);
1271 binding.pipWaiting.setVisibility(View.GONE);
1272 }
1273 } else {
1274 binding.appBarLayout.setVisibility(View.VISIBLE);
1275 binding.pipPlaceholder.setVisibility(View.GONE);
1276 }
1277 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1278 return;
1279 }
1280 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1281 binding.localVideo.setVisibility(View.GONE);
1282 binding.remoteVideoWrapper.setVisibility(View.GONE);
1283 binding.appBarLayout.setVisibility(View.GONE);
1284 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1285 binding.pipWarning.setVisibility(View.GONE);
1286 binding.pipWaiting.setVisibility(View.VISIBLE);
1287 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1288 return;
1289 }
1290 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1291 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1292 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1293 // paint local view over remote view
1294 binding.localVideo.setZOrderMediaOverlay(true);
1295 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1296 addSink(localVideoTrack.get(), binding.localVideo);
1297 } else {
1298 binding.localVideo.setVisibility(View.GONE);
1299 }
1300 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1301 if (remoteVideoTrack.isPresent()) {
1302 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1303 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1304 binding.remoteVideo.setScalingType(
1305 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1306 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1307 if (state == RtpEndUserState.CONNECTED) {
1308 binding.appBarLayout.setVisibility(View.GONE);
1309 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1310 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1311 } else {
1312 binding.appBarLayout.setVisibility(View.VISIBLE);
1313 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1314 binding.remoteVideoWrapper.setVisibility(View.GONE);
1315 }
1316 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1317 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1318 } else {
1319 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1320 }
1321 } else {
1322 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1323 binding.remoteVideoWrapper.setVisibility(View.GONE);
1324 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1325 }
1326 }
1327
1328 private Optional<VideoTrack> getLocalVideoTrack() {
1329 final JingleRtpConnection connection =
1330 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1331 if (connection == null) {
1332 return Optional.absent();
1333 }
1334 return connection.getLocalVideoTrack();
1335 }
1336
1337 private Optional<VideoTrack> getRemoteVideoTrack() {
1338 final JingleRtpConnection connection =
1339 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1340 if (connection == null) {
1341 return Optional.absent();
1342 }
1343 return connection.getRemoteVideoTrack();
1344 }
1345
1346 private void disableMicrophone(View view) {
1347 final JingleRtpConnection rtpConnection = requireRtpConnection();
1348 if (rtpConnection.setMicrophoneEnabled(false)) {
1349 updateInCallButtonConfiguration();
1350 }
1351 }
1352
1353 private void enableMicrophone(View view) {
1354 final JingleRtpConnection rtpConnection = requireRtpConnection();
1355 if (rtpConnection.setMicrophoneEnabled(true)) {
1356 updateInCallButtonConfiguration();
1357 }
1358 }
1359
1360 private void switchToEarpiece(final View view) {
1361 try {
1362 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1363 acquireProximityWakeLock();
1364 } catch (final IllegalStateException e) {
1365 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
1366 }
1367 }
1368
1369 private void switchToSpeaker(final View view) {
1370 try {
1371 requireCallIntegration().setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1372 releaseProximityWakeLock();
1373 } catch (final IllegalStateException e) {
1374 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
1375 }
1376 }
1377
1378 private void retry(final View view) {
1379 final Intent intent = getIntent();
1380 final Account account = extractAccount(intent);
1381 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1382 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1383 final String action = intent.getAction();
1384 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1385 this.rtpConnectionReference = null;
1386 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1387 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1388 }
1389
1390 private void exit(final View view) {
1391 finish();
1392 }
1393
1394 private void recordVoiceMail(final View view) {
1395 final Intent intent = getIntent();
1396 final Account account = extractAccount(intent);
1397 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1398 final Conversation conversation =
1399 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1400 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1401 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1402 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1403 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1404 launchIntent.putExtra(
1405 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1406 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1407 startActivity(launchIntent);
1408 finish();
1409 }
1410
1411 private Contact getWith() {
1412 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1413 final Account account = id.account;
1414 return account.getRoster().getContact(id.with);
1415 }
1416
1417 private JingleRtpConnection requireRtpConnection() {
1418 final JingleRtpConnection connection =
1419 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1420 if (connection == null) {
1421 throw new IllegalStateException("No RTP connection found");
1422 }
1423 return connection;
1424 }
1425
1426 private CallIntegration requireCallIntegration() {
1427 return requireOngoingRtpSession().getCallIntegration();
1428 }
1429
1430 private OngoingRtpSession requireOngoingRtpSession() {
1431 final JingleRtpConnection connection =
1432 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1433 if (connection != null) {
1434 return connection;
1435 }
1436 final Intent currentIntent = getIntent();
1437 final String withExtra =
1438 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1439 final var account = extractAccount(currentIntent);
1440 if (withExtra == null) {
1441 throw new IllegalStateException("Current intent has no EXTRA_WITH");
1442 }
1443 final var matching =
1444 xmppConnectionService
1445 .getJingleConnectionManager()
1446 .matchingProposal(account, Jid.of(withExtra));
1447 if (matching.isPresent()) {
1448 return matching.get();
1449 }
1450 throw new IllegalStateException("No matching session proposal");
1451 }
1452
1453 @Override
1454 public void onJingleRtpConnectionUpdate(
1455 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1456 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1457 if (END_CARD.contains(state)) {
1458 Log.d(Config.LOGTAG, "end card reached");
1459 releaseProximityWakeLock();
1460 runOnUiThread(
1461 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1462 }
1463 if (with.isBareJid()) {
1464 updateRtpSessionProposalState(account, with, state);
1465 return;
1466 }
1467 if (emptyReference(this.rtpConnectionReference)) {
1468 if (END_CARD.contains(state)) {
1469 Log.d(Config.LOGTAG, "not reinitializing session");
1470 return;
1471 }
1472 // this happens when going from proposed session to actual session
1473 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1474 return;
1475 }
1476 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1477 final boolean verified = requireRtpConnection().isVerified();
1478 final Set<Media> media = getMedia();
1479 lockOrientation(media);
1480 final ContentAddition contentAddition = getPendingContentAddition();
1481 final Contact contact = getWith();
1482 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1483 if (state == RtpEndUserState.ENDED) {
1484 finish();
1485 return;
1486 }
1487 runOnUiThread(
1488 () -> {
1489 updateStateDisplay(state, media, contentAddition);
1490 updateVerifiedShield(
1491 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1492 updateButtonConfiguration(state, media, contentAddition);
1493 updateVideoViews(state);
1494 updateIncomingCallScreen(state, contact);
1495 invalidateOptionsMenu();
1496 });
1497 if (END_CARD.contains(state)) {
1498 final JingleRtpConnection rtpConnection = requireRtpConnection();
1499 resetIntent(account, with, state, rtpConnection.getMedia());
1500 releaseVideoTracks(rtpConnection);
1501 this.rtpConnectionReference = null;
1502 }
1503 } else {
1504 Log.d(Config.LOGTAG, "received update for other rtp session");
1505 }
1506 }
1507
1508 @Override
1509 public void onAudioDeviceChanged(
1510 final CallIntegration.AudioDevice selectedAudioDevice,
1511 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1512 Log.d(
1513 Config.LOGTAG,
1514 "onAudioDeviceChanged in activity: selected:"
1515 + selectedAudioDevice
1516 + ", available:"
1517 + availableAudioDevices);
1518 try {
1519 final OngoingRtpSession ongoingRtpSession = requireOngoingRtpSession();
1520 final RtpEndUserState endUserState;
1521 if (ongoingRtpSession instanceof JingleRtpConnection jingleRtpConnection) {
1522 endUserState = jingleRtpConnection.getEndUserState();
1523 } else {
1524 // for session proposals all end user states are functionally the same
1525 endUserState = RtpEndUserState.RINGING;
1526 }
1527 final Set<Media> media = ongoingRtpSession.getMedia();
1528 if (END_CARD.contains(endUserState)) {
1529 Log.d(
1530 Config.LOGTAG,
1531 "onAudioDeviceChanged() nothing to do because end card has been reached");
1532 } else {
1533 if (Media.audioOnly(media)
1534 && STATES_SHOWING_SPEAKER_CONFIGURATION.contains(endUserState)) {
1535 final CallIntegration callIntegration = requireCallIntegration();
1536 updateInCallButtonConfigurationSpeaker(
1537 callIntegration.getSelectedAudioDevice(),
1538 callIntegration.getAudioDevices().size());
1539 }
1540 Log.d(
1541 Config.LOGTAG,
1542 "put proximity wake lock into proper state after device update");
1543 putProximityWakeLockInProperState(selectedAudioDevice);
1544 }
1545 } catch (final IllegalStateException e) {
1546 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1547 }
1548 }
1549
1550 @Override
1551 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1552 super.onSaveInstanceState(outState);
1553 outState.putBoolean("dialpad_visible", binding.dialpad.getVisibility() == View.VISIBLE);
1554 }
1555
1556 private void updateRtpSessionProposalState(
1557 final Account account, final Jid with, final RtpEndUserState state) {
1558 final Intent currentIntent = getIntent();
1559 final String withExtra =
1560 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1561 if (withExtra == null) {
1562 return;
1563 }
1564 final Set<Media> media = actionToMedia(currentIntent.getStringExtra(EXTRA_LAST_ACTION));
1565 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1566 runOnUiThread(
1567 () -> {
1568 updateVerifiedShield(false);
1569 updateStateDisplay(state);
1570 updateButtonConfiguration(state, media, null);
1571 updateIncomingCallScreen(state);
1572 invalidateOptionsMenu();
1573 });
1574 resetIntent(account, with, state, media);
1575 }
1576 }
1577
1578 private void resetIntent(final Bundle extras) {
1579 final Intent intent = new Intent(Intent.ACTION_VIEW);
1580 intent.putExtras(extras);
1581 setIntent(intent);
1582 }
1583
1584 private void resetIntent(
1585 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1586 final Intent intent = new Intent(Intent.ACTION_VIEW);
1587 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1588 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1589 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1590 } else {
1591 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1592 }
1593 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1594 intent.putExtra(
1595 EXTRA_LAST_ACTION,
1596 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1597 setIntent(intent);
1598 }
1599
1600 private static boolean emptyReference(final WeakReference<?> weakReference) {
1601 return weakReference == null || weakReference.get() == null;
1602 }
1603
1604 private enum Event {
1605 ON_BACKEND_CONNECTED,
1606 ON_NEW_INTENT
1607 }
1608}