1package eu.siacs.conversations.ui;
2
3import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
4
5import static java.util.Arrays.asList;
6
7import android.Manifest;
8import android.annotation.SuppressLint;
9import android.app.Activity;
10import android.app.PictureInPictureParams;
11import android.content.ActivityNotFoundException;
12import android.content.Context;
13import android.content.Intent;
14import android.content.pm.ActivityInfo;
15import android.content.pm.PackageManager;
16import android.opengl.GLException;
17import android.os.Build;
18import android.os.Bundle;
19import android.os.Handler;
20import android.os.PowerManager;
21import android.os.SystemClock;
22import android.util.Log;
23import android.util.Rational;
24import android.view.KeyEvent;
25import android.view.Menu;
26import android.view.MenuItem;
27import android.view.View;
28import android.view.WindowManager;
29import android.widget.Toast;
30
31import androidx.annotation.NonNull;
32import androidx.annotation.Nullable;
33import androidx.annotation.RequiresApi;
34import androidx.annotation.StringRes;
35import androidx.databinding.DataBindingUtil;
36
37import com.google.common.base.Optional;
38import com.google.common.base.Preconditions;
39import com.google.common.base.Throwables;
40import com.google.common.collect.ImmutableList;
41import com.google.common.collect.ImmutableSet;
42import com.google.common.util.concurrent.FutureCallback;
43import com.google.common.util.concurrent.Futures;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.services.CallIntegration;
53import eu.siacs.conversations.services.CallIntegrationConnectionService;
54import eu.siacs.conversations.services.XmppConnectionService;
55import eu.siacs.conversations.ui.util.AvatarWorkerTask;
56import eu.siacs.conversations.ui.util.MainThreadExecutor;
57import eu.siacs.conversations.ui.util.Rationals;
58import eu.siacs.conversations.utils.PermissionUtils;
59import eu.siacs.conversations.utils.TimeFrameUtils;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
62import eu.siacs.conversations.xmpp.jingle.ContentAddition;
63import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
64import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
65import eu.siacs.conversations.xmpp.jingle.Media;
66import eu.siacs.conversations.xmpp.jingle.RtpCapability;
67import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
68
69import org.webrtc.RendererCommon;
70import org.webrtc.SurfaceViewRenderer;
71import org.webrtc.VideoTrack;
72
73import java.lang.ref.WeakReference;
74import java.util.Arrays;
75import java.util.Collections;
76import java.util.List;
77import java.util.Set;
78
79public class RtpSessionActivity extends XmppActivity
80 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
81 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
82
83 public static final String EXTRA_WITH = "with";
84 public static final String EXTRA_SESSION_ID = "session_id";
85 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
86 public static final String EXTRA_LAST_ACTION = "last_action";
87 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
88 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
89 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
90
91 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
92
93 private static final List<RtpEndUserState> END_CARD =
94 Arrays.asList(
95 RtpEndUserState.APPLICATION_ERROR,
96 RtpEndUserState.SECURITY_ERROR,
97 RtpEndUserState.DECLINED_OR_BUSY,
98 RtpEndUserState.CONTACT_OFFLINE,
99 RtpEndUserState.CONNECTIVITY_ERROR,
100 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
101 RtpEndUserState.RETRACTED);
102 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
103 Arrays.asList(
104 RtpEndUserState.APPLICATION_ERROR,
105 RtpEndUserState.CONNECTIVITY_ERROR,
106 RtpEndUserState.SECURITY_ERROR);
107 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
108 Arrays.asList(
109 RtpEndUserState.CONNECTING,
110 RtpEndUserState.CONNECTED,
111 RtpEndUserState.RECONNECTING,
112 RtpEndUserState.INCOMING_CONTENT_ADD);
113 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
114 Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING);
115 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
116 Arrays.asList(
117 RtpEndUserState.ACCEPTING_CALL,
118 RtpEndUserState.CONNECTING,
119 RtpEndUserState.RECONNECTING);
120 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
121 private static final int REQUEST_ACCEPT_CALL = 0x1111;
122 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
123 private static final int REQUEST_ADD_CONTENT = 0x1113;
124 private WeakReference<JingleRtpConnection> rtpConnectionReference;
125
126 private ActivityRtpSessionBinding binding;
127 private PowerManager.WakeLock mProximityWakeLock;
128
129 private final Handler mHandler = new Handler();
130 private final Runnable mTickExecutor =
131 new Runnable() {
132 @Override
133 public void run() {
134 updateCallDuration();
135 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
136 }
137 };
138
139 public static Set<Media> actionToMedia(final String action) {
140 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
141 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
142 } else {
143 return ImmutableSet.of(Media.AUDIO);
144 }
145 }
146
147 private static void addSink(
148 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
149 try {
150 videoTrack.addSink(surfaceViewRenderer);
151 } catch (final IllegalStateException e) {
152 Log.e(
153 Config.LOGTAG,
154 "possible race condition on trying to display video track. ignoring",
155 e);
156 }
157 }
158
159 @Override
160 public void onCreate(Bundle savedInstanceState) {
161 super.onCreate(savedInstanceState);
162 getWindow()
163 .addFlags(
164 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
165 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
166 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
167 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
168 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
169 setSupportActionBar(binding.toolbar);
170 }
171
172 @Override
173 public boolean onCreateOptionsMenu(final Menu menu) {
174 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
175 final MenuItem help = menu.findItem(R.id.action_help);
176 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
177 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
178 help.setVisible(Config.HELP != null && isHelpButtonVisible());
179 gotoChat.setVisible(isSwitchToConversationVisible());
180 switchToVideo.setVisible(isSwitchToVideoVisible());
181 return super.onCreateOptionsMenu(menu);
182 }
183
184 @Override
185 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
186 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
187 if (xmppConnectionService != null) {
188 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
189 return true;
190 }
191 }
192 }
193 return super.onKeyDown(keyCode, event);
194 }
195
196 private boolean isHelpButtonVisible() {
197 try {
198 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
199 } catch (IllegalStateException e) {
200 final Intent intent = getIntent();
201 final String state =
202 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
203 if (state != null) {
204 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
205 } else {
206 return false;
207 }
208 }
209 }
210
211 private boolean isSwitchToConversationVisible() {
212 final JingleRtpConnection connection =
213 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
214 return connection != null
215 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
216 }
217
218 private boolean isSwitchToVideoVisible() {
219 final JingleRtpConnection connection =
220 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
221 if (connection == null) {
222 return false;
223 }
224 return connection.isSwitchToVideoAvailable();
225 }
226
227 private void switchToConversation() {
228 final Contact contact = getWith();
229 final Conversation conversation =
230 xmppConnectionService.findOrCreateConversation(
231 contact.getAccount(), contact.getJid(), false, true);
232 switchToConversation(conversation);
233 }
234
235 public boolean onOptionsItemSelected(final MenuItem item) {
236 final var itemItem = item.getItemId();
237 if (itemItem == R.id.action_help) {
238 launchHelpInBrowser();
239 return true;
240 } else if (itemItem == R.id.action_goto_chat) {
241 switchToConversation();
242 return true;
243 } else if (itemItem == R.id.action_switch_to_video) {
244 requestPermissionAndSwitchToVideo();
245 return true;
246 } else {
247 return super.onOptionsItemSelected(item);
248 }
249 }
250
251 private void launchHelpInBrowser() {
252 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
253 try {
254 startActivity(intent);
255 } catch (final ActivityNotFoundException e) {
256 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
257 .show();
258 }
259 }
260
261 private void endCall(View view) {
262 endCall();
263 }
264
265 private void endCall() {
266 if (this.rtpConnectionReference == null) {
267 retractSessionProposal();
268 finish();
269 } else {
270 requireRtpConnection().endCall();
271 }
272 }
273
274 private void retractSessionProposal() {
275 final Intent intent = getIntent();
276 final String action = intent.getAction();
277 final Account account = extractAccount(intent);
278 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
279 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
280 if (!Intent.ACTION_VIEW.equals(action)
281 || state == null
282 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
283 resetIntent(
284 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
285 }
286 xmppConnectionService
287 .getJingleConnectionManager()
288 .retractSessionProposal(account, with.asBareJid());
289 }
290
291 private void rejectCall(View view) {
292 requireRtpConnection().rejectCall();
293 finish();
294 }
295
296 private void acceptCall(View view) {
297 requestPermissionsAndAcceptCall();
298 }
299
300 private void acceptContentAdd() {
301 try {
302 requireRtpConnection()
303 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
304 } catch (final IllegalStateException e) {
305 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
306 }
307 }
308
309 private void requestPermissionAndSwitchToVideo() {
310 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
311 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
312 switchToVideo();
313 }
314 }
315
316 private void switchToVideo() {
317 try {
318 requireRtpConnection().addMedia(Media.VIDEO);
319 } catch (final IllegalStateException e) {
320 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
321 }
322 }
323
324 private void acceptContentAdd(final ContentAddition contentAddition) {
325 if (contentAddition == null
326 || contentAddition.direction != ContentAddition.Direction.INCOMING) {
327 Log.d(Config.LOGTAG, "ignore press on content-accept button");
328 return;
329 }
330 requestPermissionAndAcceptContentAdd(contentAddition);
331 }
332
333 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
334 final List<String> permissions = permissions(contentAddition.media());
335 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
336 try {
337 requireRtpConnection().acceptContentAdd(contentAddition.summary);
338 } catch (final IllegalStateException e) {
339 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
340 }
341 }
342 }
343
344 private void rejectContentAdd(final View view) {
345 requireRtpConnection().rejectContentAdd();
346 }
347
348 private void requestPermissionsAndAcceptCall() {
349 final List<String> permissions = permissions(getMedia());
350 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
351 putScreenInCallMode();
352 checkRecorderAndAcceptCall();
353 }
354 }
355
356 private List<String> permissions(final Set<Media> media) {
357 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
358 if (media.contains(Media.VIDEO)) {
359 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
360 } else {
361 permissions.add(Manifest.permission.RECORD_AUDIO);
362 }
363 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
364 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
365 }
366 return permissions.build();
367 }
368
369 private void checkRecorderAndAcceptCall() {
370 checkMicrophoneAvailabilityAsync();
371 try {
372 requireRtpConnection().acceptCall();
373 } catch (final IllegalStateException e) {
374 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
375 }
376 }
377
378 private void checkMicrophoneAvailabilityAsync() {
379 new Thread(new MicrophoneAvailabilityCheck(this)).start();
380 }
381
382 private static class MicrophoneAvailabilityCheck implements Runnable {
383
384 private final WeakReference<Activity> activityReference;
385
386 private MicrophoneAvailabilityCheck(final Activity activity) {
387 this.activityReference = new WeakReference<>(activity);
388 }
389
390 @Override
391 public void run() {
392 final long start = SystemClock.elapsedRealtime();
393 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
394 final long stop = SystemClock.elapsedRealtime();
395 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
396 if (isMicrophoneAvailable) {
397 return;
398 }
399 final Activity activity = activityReference.get();
400 if (activity == null) {
401 return;
402 }
403 activity.runOnUiThread(
404 () ->
405 Toast.makeText(
406 activity,
407 R.string.microphone_unavailable,
408 Toast.LENGTH_LONG)
409 .show());
410 }
411 }
412
413 private void putScreenInCallMode() {
414 putScreenInCallMode(requireRtpConnection().getMedia());
415 }
416
417 private void putScreenInCallMode(final Set<Media> media) {
418 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
419 if (Media.audioOnly(media)) {
420 final JingleRtpConnection rtpConnection =
421 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
422 final CallIntegration callIntegration =
423 rtpConnection == null ? null : rtpConnection.getCallIntegration();
424 if (callIntegration == null
425 || callIntegration.getSelectedAudioDevice()
426 == CallIntegration.AudioDevice.EARPIECE) {
427 acquireProximityWakeLock();
428 }
429 }
430 lockOrientation(media);
431 }
432
433 private void lockOrientation(final Set<Media> media) {
434 if (Media.audioOnly(media)) {
435 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
436 } else {
437 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
438 }
439 }
440
441 @SuppressLint("WakelockTimeout")
442 private void acquireProximityWakeLock() {
443 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
444 if (powerManager == null) {
445 Log.e(Config.LOGTAG, "power manager not available");
446 return;
447 }
448 if (isFinishing()) {
449 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
450 return;
451 }
452 if (this.mProximityWakeLock == null) {
453 this.mProximityWakeLock =
454 powerManager.newWakeLock(
455 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
456 }
457 if (!this.mProximityWakeLock.isHeld()) {
458 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
459 this.mProximityWakeLock.acquire();
460 }
461 }
462
463 private void releaseProximityWakeLock() {
464 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
465 Log.d(Config.LOGTAG, "releasing proximity wake lock");
466 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
467 this.mProximityWakeLock = null;
468 }
469 }
470
471 private void putProximityWakeLockInProperState(final CallIntegration.AudioDevice audioDevice) {
472 if (audioDevice == CallIntegration.AudioDevice.EARPIECE) {
473 acquireProximityWakeLock();
474 } else {
475 releaseProximityWakeLock();
476 }
477 }
478
479 @Override
480 protected void refreshUiReal() {}
481
482 @Override
483 public void onNewIntent(final Intent intent) {
484 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
485 super.onNewIntent(intent);
486 setIntent(intent);
487 if (xmppConnectionService == null) {
488 Log.d(
489 Config.LOGTAG,
490 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
491 return;
492 }
493 final Account account = extractAccount(intent);
494 final String action = intent.getAction();
495 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
496 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
497 if (sessionId != null) {
498 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
499 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
500 return;
501 }
502 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
503 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
504 requestPermissionsAndAcceptCall();
505 resetIntent(intent.getExtras());
506 }
507 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
508 proposeJingleRtpSession(account, with, actionToMedia(action));
509 setWith(account.getRoster().getContact(with), null);
510 } else {
511 throw new IllegalStateException("received onNewIntent without sessionId");
512 }
513 }
514
515 @Override
516 void onBackendConnected() {
517 final Intent intent = getIntent();
518 final String action = intent.getAction();
519 final Account account = extractAccount(intent);
520 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
521 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
522 if (sessionId != null) {
523 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
524 return;
525 }
526 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
527 Log.d(Config.LOGTAG, "intent action was accept");
528 requestPermissionsAndAcceptCall();
529 resetIntent(intent.getExtras());
530 }
531 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
532 proposeJingleRtpSession(account, with, actionToMedia(action));
533 setWith(account.getRoster().getContact(with), null);
534 } else if (Intent.ACTION_VIEW.equals(action)) {
535 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
536 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
537 final Set<Media> media = actionToMedia(lastAction);
538 final RtpEndUserState state =
539 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
540 if (state != null) {
541 Log.d(Config.LOGTAG, "restored last state from intent extra");
542 updateButtonConfiguration(state);
543 updateVerifiedShield(false);
544 updateStateDisplay(state);
545 updateIncomingCallScreen(state);
546 invalidateOptionsMenu();
547 }
548 setWith(account.getRoster().getContact(with), state);
549 if (xmppConnectionService
550 .getJingleConnectionManager()
551 .fireJingleRtpConnectionStateUpdates()) {
552 return;
553 }
554 if (END_CARD.contains(state)) {
555 return;
556 }
557 if (xmppConnectionService
558 .getJingleConnectionManager()
559 .hasMatchingProposal(account, with)) {
560 putScreenInCallMode(media);
561 return;
562 }
563 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
564 finish();
565 }
566 }
567
568 private void setWidth(final RtpEndUserState state) {
569 setWith(getWith(), state);
570 }
571
572 private void setWith(final Contact contact, final RtpEndUserState state) {
573 binding.with.setText(contact.getDisplayName());
574 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
575 .contains(state)) {
576 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
577 binding.withJid.setVisibility(View.VISIBLE);
578 } else {
579 binding.withJid.setVisibility(View.GONE);
580 }
581 }
582
583 private void proposeJingleRtpSession(
584 final Account account, final Jid with, final Set<Media> media) {
585 checkMicrophoneAvailabilityAsync();
586 if (with.isBareJid()) {
587 xmppConnectionService
588 .getJingleConnectionManager()
589 .proposeJingleRtpSession(account, with, media);
590 } else {
591 throw new IllegalStateException(
592 "We should not be initializing direct calls from the RtpSessionActivity. Go through CallIntegrationConnectionService.placeCall instead!");
593 }
594 putScreenInCallMode(media);
595 }
596
597 @Override
598 public void onRequestPermissionsResult(
599 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
600 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
601 final PermissionUtils.PermissionResult permissionResult =
602 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
603 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
604 if (requestCode == REQUEST_ACCEPT_CALL) {
605 checkRecorderAndAcceptCall();
606 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
607 acceptContentAdd();
608 } else if (requestCode == REQUEST_ADD_CONTENT) {
609 switchToVideo();
610 }
611 } else {
612 @StringRes int res;
613 final String firstDenied =
614 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
615 if (firstDenied == null) {
616 return;
617 }
618 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
619 res = R.string.no_microphone_permission;
620 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
621 res = R.string.no_camera_permission;
622 } else {
623 throw new IllegalStateException("Invalid permission result request");
624 }
625 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
626 .show();
627 }
628 }
629
630 @Override
631 public void onStart() {
632 super.onStart();
633 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
634 this.binding.remoteVideo.setOnAspectRatioChanged(this);
635 }
636
637 @Override
638 public void onStop() {
639 mHandler.removeCallbacks(mTickExecutor);
640 binding.remoteVideo.release();
641 binding.remoteVideo.setOnAspectRatioChanged(null);
642 binding.localVideo.release();
643 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
644 final JingleRtpConnection jingleRtpConnection =
645 weakReference == null ? null : weakReference.get();
646 if (jingleRtpConnection != null) {
647 releaseVideoTracks(jingleRtpConnection);
648 }
649 releaseProximityWakeLock();
650 super.onStop();
651 }
652
653 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
654 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
655 if (remoteVideo.isPresent()) {
656 remoteVideo.get().removeSink(binding.remoteVideo);
657 }
658 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
659 if (localVideo.isPresent()) {
660 localVideo.get().removeSink(binding.localVideo);
661 }
662 }
663
664 @Override
665 public void onBackPressed() {
666 if (isConnected()) {
667 if (switchToPictureInPicture()) {
668 return;
669 }
670 } else {
671 endCall();
672 }
673 super.onBackPressed();
674 }
675
676 @Override
677 public void onUserLeaveHint() {
678 super.onUserLeaveHint();
679 if (switchToPictureInPicture()) {
680 return;
681 }
682 // TODO apparently this method is not getting called on Android 10 when using the task
683 // switcher
684 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
685 retractSessionProposal();
686 }
687 }
688
689 private boolean isConnected() {
690 final JingleRtpConnection connection =
691 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
692 final RtpEndUserState endUserState =
693 connection == null ? null : connection.getEndUserState();
694 return STATES_CONSIDERED_CONNECTED.contains(endUserState)
695 || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
696 }
697
698 private boolean switchToPictureInPicture() {
699 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
700 if (shouldBePictureInPicture()) {
701 startPictureInPicture();
702 return true;
703 }
704 }
705 return false;
706 }
707
708 @RequiresApi(api = Build.VERSION_CODES.O)
709 private void startPictureInPicture() {
710 try {
711 final Rational rational = this.binding.remoteVideo.getAspectRatio();
712 final Rational clippedRational = Rationals.clip(rational);
713 Log.d(
714 Config.LOGTAG,
715 "suggested rational " + rational + ". clipped to " + clippedRational);
716 enterPictureInPictureMode(
717 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
718 } catch (final IllegalStateException e) {
719 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
720 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
721 }
722 }
723
724 @Override
725 public void onAspectRatioChanged(final Rational rational) {
726 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
727 final Rational clippedRational = Rationals.clip(rational);
728 Log.d(
729 Config.LOGTAG,
730 "suggested rational after aspect ratio change "
731 + rational
732 + ". clipped to "
733 + clippedRational);
734 setPictureInPictureParams(
735 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
736 }
737 }
738
739 private boolean deviceSupportsPictureInPicture() {
740 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
741 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
742 } else {
743 return false;
744 }
745 }
746
747 private boolean shouldBePictureInPicture() {
748 try {
749 final JingleRtpConnection rtpConnection = requireRtpConnection();
750 return rtpConnection.getMedia().contains(Media.VIDEO)
751 && Arrays.asList(
752 RtpEndUserState.ACCEPTING_CALL,
753 RtpEndUserState.CONNECTING,
754 RtpEndUserState.CONNECTED)
755 .contains(rtpConnection.getEndUserState());
756 } catch (final IllegalStateException e) {
757 return false;
758 }
759 }
760
761 private boolean initializeActivityWithRunningRtpSession(
762 final Account account, Jid with, String sessionId) {
763 final WeakReference<JingleRtpConnection> reference =
764 xmppConnectionService
765 .getJingleConnectionManager()
766 .findJingleRtpConnection(account, with, sessionId);
767 if (reference == null || reference.get() == null) {
768 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
769 xmppConnectionService
770 .getJingleConnectionManager()
771 .getTerminalSessionState(with, sessionId);
772 if (terminatedRtpSession == null) {
773 throw new IllegalStateException(
774 "failed to initialize activity with running rtp session. session not found");
775 }
776 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
777 return true;
778 }
779 this.rtpConnectionReference = reference;
780 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
781 final boolean verified = requireRtpConnection().isVerified();
782 if (currentState == RtpEndUserState.ENDED) {
783 finish();
784 return true;
785 }
786 final Set<Media> media = getMedia();
787 final ContentAddition contentAddition = getPendingContentAddition();
788 if (currentState == RtpEndUserState.INCOMING_CALL) {
789 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
790 }
791 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
792 requireRtpConnection().getState())) {
793 putScreenInCallMode();
794 }
795 setWidth(currentState);
796 updateVideoViews(currentState);
797 updateStateDisplay(currentState, media, contentAddition);
798 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
799 updateButtonConfiguration(currentState, media, contentAddition);
800 updateIncomingCallScreen(currentState);
801 invalidateOptionsMenu();
802 return false;
803 }
804
805 private void initializeWithTerminatedSessionState(
806 final Account account,
807 final Jid with,
808 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
809 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
810 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
811 finish();
812 return;
813 }
814 final RtpEndUserState state = terminatedRtpSession.state;
815 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
816 updateButtonConfiguration(state);
817 updateStateDisplay(state);
818 updateIncomingCallScreen(state);
819 updateCallDuration();
820 updateVerifiedShield(false);
821 invalidateOptionsMenu();
822 setWith(account.getRoster().getContact(with), state);
823 }
824
825 private void reInitializeActivityWithRunningRtpSession(
826 final Account account, Jid with, String sessionId) {
827 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
828 resetIntent(account, with, sessionId);
829 }
830
831 private void resetIntent(final Account account, final Jid with, final String sessionId) {
832 final Intent intent = new Intent(Intent.ACTION_VIEW);
833 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
834 intent.putExtra(EXTRA_WITH, with.toEscapedString());
835 intent.putExtra(EXTRA_SESSION_ID, sessionId);
836 setIntent(intent);
837 }
838
839 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
840 surfaceViewRenderer.setVisibility(View.VISIBLE);
841 try {
842 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
843 } catch (final IllegalStateException ignored) {
844 // SurfaceViewRenderer was already initialized
845 } catch (final RuntimeException e) {
846 if (Throwables.getRootCause(e) instanceof GLException glException) {
847 Log.w(Config.LOGTAG, "could not set up hardware renderer", glException);
848 }
849 }
850 surfaceViewRenderer.setEnableHardwareScaler(true);
851 }
852
853 private void updateStateDisplay(final RtpEndUserState state) {
854 updateStateDisplay(state, Collections.emptySet(), null);
855 }
856
857 private void updateStateDisplay(
858 final RtpEndUserState state,
859 final Set<Media> media,
860 final ContentAddition contentAddition) {
861 switch (state) {
862 case INCOMING_CALL -> {
863 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
864 if (media.contains(Media.VIDEO)) {
865 setTitle(R.string.rtp_state_incoming_video_call);
866 } else {
867 setTitle(R.string.rtp_state_incoming_call);
868 }
869 }
870 case INCOMING_CONTENT_ADD -> {
871 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
872 setTitle(R.string.rtp_state_content_add_video);
873 } else {
874 setTitle(R.string.rtp_state_content_add);
875 }
876 }
877 case CONNECTING -> setTitle(R.string.rtp_state_connecting);
878 case CONNECTED -> setTitle(R.string.rtp_state_connected);
879 case RECONNECTING -> setTitle(R.string.rtp_state_reconnecting);
880 case ACCEPTING_CALL -> setTitle(R.string.rtp_state_accepting_call);
881 case ENDING_CALL -> setTitle(R.string.rtp_state_ending_call);
882 case FINDING_DEVICE -> setTitle(R.string.rtp_state_finding_device);
883 case RINGING -> setTitle(R.string.rtp_state_ringing);
884 case DECLINED_OR_BUSY -> setTitle(R.string.rtp_state_declined_or_busy);
885 case CONTACT_OFFLINE -> setTitle(R.string.rtp_state_contact_offline);
886 case CONNECTIVITY_ERROR -> setTitle(R.string.rtp_state_connectivity_error);
887 case CONNECTIVITY_LOST_ERROR -> setTitle(R.string.rtp_state_connectivity_lost_error);
888 case RETRACTED -> setTitle(R.string.rtp_state_retracted);
889 case APPLICATION_ERROR -> setTitle(R.string.rtp_state_application_failure);
890 case SECURITY_ERROR -> setTitle(R.string.rtp_state_security_error);
891 case ENDED -> throw new IllegalStateException(
892 "Activity should have called finishAndReleaseWakeLock();");
893 default -> throw new IllegalStateException(
894 String.format("State %s has not been handled in UI", state));
895 }
896 }
897
898 private void updateVerifiedShield(final boolean verified) {
899 if (isPictureInPicture()) {
900 this.binding.verified.setVisibility(View.GONE);
901 return;
902 }
903 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
904 }
905
906 private void updateIncomingCallScreen(final RtpEndUserState state) {
907 updateIncomingCallScreen(state, null);
908 }
909
910 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
911 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
912 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
913 if (show) {
914 binding.contactPhoto.setVisibility(View.VISIBLE);
915 if (contact == null) {
916 AvatarWorkerTask.loadAvatar(
917 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
918 } else {
919 AvatarWorkerTask.loadAvatar(
920 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
921 }
922 } else {
923 binding.contactPhoto.setVisibility(View.GONE);
924 }
925 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
926 binding.usingAccount.setVisibility(View.VISIBLE);
927 binding.usingAccount.setText(
928 getString(
929 R.string.using_account,
930 account.getJid().asBareJid().toEscapedString()));
931 } else {
932 binding.usingAccount.setVisibility(View.GONE);
933 binding.contactPhoto.setVisibility(View.GONE);
934 }
935 }
936
937 private Set<Media> getMedia() {
938 return requireRtpConnection().getMedia();
939 }
940
941 public ContentAddition getPendingContentAddition() {
942 return requireRtpConnection().getPendingContentAddition();
943 }
944
945 private void updateButtonConfiguration(final RtpEndUserState state) {
946 updateButtonConfiguration(state, Collections.emptySet(), null);
947 }
948
949 @SuppressLint("RestrictedApi")
950 private void updateButtonConfiguration(
951 final RtpEndUserState state,
952 final Set<Media> media,
953 final ContentAddition contentAddition) {
954 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
955 this.binding.rejectCall.setVisibility(View.INVISIBLE);
956 this.binding.endCall.setVisibility(View.INVISIBLE);
957 this.binding.acceptCall.setVisibility(View.INVISIBLE);
958 } else if (state == RtpEndUserState.INCOMING_CALL) {
959 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
960 this.binding.rejectCall.setOnClickListener(this::rejectCall);
961 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
962 this.binding.rejectCall.setVisibility(View.VISIBLE);
963 this.binding.endCall.setVisibility(View.INVISIBLE);
964 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
965 this.binding.acceptCall.setOnClickListener(this::acceptCall);
966 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
967 this.binding.acceptCall.setVisibility(View.VISIBLE);
968 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
969 this.binding.rejectCall.setContentDescription(
970 getString(R.string.reject_switch_to_video));
971 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
972 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
973 this.binding.rejectCall.setVisibility(View.VISIBLE);
974 this.binding.endCall.setVisibility(View.INVISIBLE);
975 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
976 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
977 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
978 this.binding.acceptCall.setVisibility(View.VISIBLE);
979 } else if (asList(RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.CONTACT_OFFLINE)
980 .contains(state)) {
981 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
982 this.binding.rejectCall.setOnClickListener(this::exit);
983 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
984 this.binding.rejectCall.setVisibility(View.VISIBLE);
985 this.binding.endCall.setVisibility(View.INVISIBLE);
986 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
987 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
988 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
989 this.binding.acceptCall.setVisibility(View.VISIBLE);
990 } else if (asList(
991 RtpEndUserState.CONNECTIVITY_ERROR,
992 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
993 RtpEndUserState.APPLICATION_ERROR,
994 RtpEndUserState.RETRACTED,
995 RtpEndUserState.SECURITY_ERROR)
996 .contains(state)) {
997 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
998 this.binding.rejectCall.setOnClickListener(this::exit);
999 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1000 this.binding.rejectCall.setVisibility(View.VISIBLE);
1001 this.binding.endCall.setVisibility(View.INVISIBLE);
1002 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1003 this.binding.acceptCall.setOnClickListener(this::retry);
1004 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1005 this.binding.acceptCall.setVisibility(View.VISIBLE);
1006 } else {
1007 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1008 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1009 this.binding.endCall.setOnClickListener(this::endCall);
1010 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1011 this.binding.endCall.setVisibility(View.VISIBLE);
1012 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1013 }
1014 updateInCallButtonConfiguration(state, media);
1015 }
1016
1017 private boolean isPictureInPicture() {
1018 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1019 return isInPictureInPictureMode();
1020 } else {
1021 return false;
1022 }
1023 }
1024
1025 private void updateInCallButtonConfiguration() {
1026 updateInCallButtonConfiguration(
1027 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1028 }
1029
1030 @SuppressLint("RestrictedApi")
1031 private void updateInCallButtonConfiguration(
1032 final RtpEndUserState state, final Set<Media> media) {
1033 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1034 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1035 if (media.contains(Media.VIDEO)) {
1036 final JingleRtpConnection rtpConnection = requireRtpConnection();
1037 updateInCallButtonConfigurationVideo(
1038 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1039 } else {
1040 final CallIntegration callIntegration = requireRtpConnection().getCallIntegration();
1041 updateInCallButtonConfigurationSpeaker(
1042 callIntegration.getSelectedAudioDevice(),
1043 callIntegration.getAudioDevices().size());
1044 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1045 }
1046 if (media.contains(Media.AUDIO)) {
1047 updateInCallButtonConfigurationMicrophone(
1048 requireRtpConnection().isMicrophoneEnabled());
1049 } else {
1050 this.binding.inCallActionLeft.setVisibility(View.GONE);
1051 }
1052 } else {
1053 this.binding.inCallActionLeft.setVisibility(View.GONE);
1054 this.binding.inCallActionRight.setVisibility(View.GONE);
1055 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1056 }
1057 }
1058
1059 @SuppressLint("RestrictedApi")
1060 private void updateInCallButtonConfigurationSpeaker(
1061 final CallIntegration.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1062 switch (selectedAudioDevice) {
1063 case EARPIECE -> {
1064 this.binding.inCallActionRight.setImageResource(
1065 R.drawable.ic_volume_off_black_24dp);
1066 if (numberOfChoices >= 2) {
1067 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1068 } else {
1069 this.binding.inCallActionRight.setOnClickListener(null);
1070 this.binding.inCallActionRight.setClickable(false);
1071 }
1072 }
1073 case WIRED_HEADSET -> {
1074 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1075 this.binding.inCallActionRight.setOnClickListener(null);
1076 this.binding.inCallActionRight.setClickable(false);
1077 }
1078 case SPEAKER_PHONE -> {
1079 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1080 if (numberOfChoices >= 2) {
1081 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1082 } else {
1083 this.binding.inCallActionRight.setOnClickListener(null);
1084 this.binding.inCallActionRight.setClickable(false);
1085 }
1086 }
1087 case BLUETOOTH -> {
1088 this.binding.inCallActionRight.setImageResource(
1089 R.drawable.ic_bluetooth_audio_black_24dp);
1090 this.binding.inCallActionRight.setOnClickListener(null);
1091 this.binding.inCallActionRight.setClickable(false);
1092 }
1093 }
1094 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1095 }
1096
1097 @SuppressLint("RestrictedApi")
1098 private void updateInCallButtonConfigurationVideo(
1099 final boolean videoEnabled, final boolean isCameraSwitchable) {
1100 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1101 if (isCameraSwitchable) {
1102 this.binding.inCallActionFarRight.setImageResource(
1103 R.drawable.ic_flip_camera_android_black_24dp);
1104 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1105 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1106 } else {
1107 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1108 }
1109 if (videoEnabled) {
1110 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1111 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1112 } else {
1113 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1114 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1115 }
1116 }
1117
1118 private void switchCamera(final View view) {
1119 Futures.addCallback(
1120 requireRtpConnection().switchCamera(),
1121 new FutureCallback<>() {
1122 @Override
1123 public void onSuccess(@Nullable Boolean isFrontCamera) {
1124 binding.localVideo.setMirror(Boolean.TRUE.equals(isFrontCamera));
1125 }
1126
1127 @Override
1128 public void onFailure(@NonNull final Throwable throwable) {
1129 Log.d(
1130 Config.LOGTAG,
1131 "could not switch camera",
1132 Throwables.getRootCause(throwable));
1133 Toast.makeText(
1134 RtpSessionActivity.this,
1135 R.string.could_not_switch_camera,
1136 Toast.LENGTH_LONG)
1137 .show();
1138 }
1139 },
1140 MainThreadExecutor.getInstance());
1141 }
1142
1143 private void enableVideo(View view) {
1144 try {
1145 requireRtpConnection().setVideoEnabled(true);
1146 } catch (final IllegalStateException e) {
1147 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1148 return;
1149 }
1150 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1151 }
1152
1153 private void disableVideo(View view) {
1154 final JingleRtpConnection rtpConnection = requireRtpConnection();
1155 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1156 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1157 rtpConnection.retractContentAdd();
1158 return;
1159 }
1160 requireRtpConnection().setVideoEnabled(false);
1161 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1162 }
1163
1164 @SuppressLint("RestrictedApi")
1165 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1166 if (microphoneEnabled) {
1167 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1168 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1169 } else {
1170 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1171 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1172 }
1173 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1174 }
1175
1176 private void updateCallDuration() {
1177 final JingleRtpConnection connection =
1178 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1179 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1180 this.binding.duration.setVisibility(View.GONE);
1181 return;
1182 }
1183 if (connection.zeroDuration()) {
1184 this.binding.duration.setVisibility(View.GONE);
1185 } else {
1186 this.binding.duration.setText(
1187 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1188 this.binding.duration.setVisibility(View.VISIBLE);
1189 }
1190 }
1191
1192 private void updateVideoViews(final RtpEndUserState state) {
1193 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1194 binding.localVideo.setVisibility(View.GONE);
1195 binding.localVideo.release();
1196 binding.remoteVideoWrapper.setVisibility(View.GONE);
1197 binding.remoteVideo.release();
1198 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1199 if (isPictureInPicture()) {
1200 binding.appBarLayout.setVisibility(View.GONE);
1201 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1202 if (Arrays.asList(
1203 RtpEndUserState.APPLICATION_ERROR,
1204 RtpEndUserState.CONNECTIVITY_ERROR,
1205 RtpEndUserState.SECURITY_ERROR)
1206 .contains(state)) {
1207 binding.pipWarning.setVisibility(View.VISIBLE);
1208 binding.pipWaiting.setVisibility(View.GONE);
1209 } else {
1210 binding.pipWarning.setVisibility(View.GONE);
1211 binding.pipWaiting.setVisibility(View.GONE);
1212 }
1213 } else {
1214 binding.appBarLayout.setVisibility(View.VISIBLE);
1215 binding.pipPlaceholder.setVisibility(View.GONE);
1216 }
1217 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1218 return;
1219 }
1220 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1221 binding.localVideo.setVisibility(View.GONE);
1222 binding.remoteVideoWrapper.setVisibility(View.GONE);
1223 binding.appBarLayout.setVisibility(View.GONE);
1224 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1225 binding.pipWarning.setVisibility(View.GONE);
1226 binding.pipWaiting.setVisibility(View.VISIBLE);
1227 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1228 return;
1229 }
1230 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1231 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1232 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1233 // paint local view over remote view
1234 binding.localVideo.setZOrderMediaOverlay(true);
1235 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1236 addSink(localVideoTrack.get(), binding.localVideo);
1237 } else {
1238 binding.localVideo.setVisibility(View.GONE);
1239 }
1240 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1241 if (remoteVideoTrack.isPresent()) {
1242 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1243 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1244 binding.remoteVideo.setScalingType(
1245 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1246 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1247 if (state == RtpEndUserState.CONNECTED) {
1248 binding.appBarLayout.setVisibility(View.GONE);
1249 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1250 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1251 } else {
1252 binding.appBarLayout.setVisibility(View.VISIBLE);
1253 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1254 binding.remoteVideoWrapper.setVisibility(View.GONE);
1255 }
1256 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1257 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1258 } else {
1259 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1260 }
1261 } else {
1262 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1263 binding.remoteVideoWrapper.setVisibility(View.GONE);
1264 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1265 }
1266 }
1267
1268 private Optional<VideoTrack> getLocalVideoTrack() {
1269 final JingleRtpConnection connection =
1270 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1271 if (connection == null) {
1272 return Optional.absent();
1273 }
1274 return connection.getLocalVideoTrack();
1275 }
1276
1277 private Optional<VideoTrack> getRemoteVideoTrack() {
1278 final JingleRtpConnection connection =
1279 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1280 if (connection == null) {
1281 return Optional.absent();
1282 }
1283 return connection.getRemoteVideoTrack();
1284 }
1285
1286 private void disableMicrophone(View view) {
1287 final JingleRtpConnection rtpConnection = requireRtpConnection();
1288 if (rtpConnection.setMicrophoneEnabled(false)) {
1289 updateInCallButtonConfiguration();
1290 }
1291 }
1292
1293 private void enableMicrophone(View view) {
1294 final JingleRtpConnection rtpConnection = requireRtpConnection();
1295 if (rtpConnection.setMicrophoneEnabled(true)) {
1296 updateInCallButtonConfiguration();
1297 }
1298 }
1299
1300 private void switchToEarpiece(View view) {
1301 requireRtpConnection()
1302 .getCallIntegration()
1303 .setAudioDevice(CallIntegration.AudioDevice.EARPIECE);
1304 acquireProximityWakeLock();
1305 }
1306
1307 private void switchToSpeaker(View view) {
1308 requireRtpConnection()
1309 .getCallIntegration()
1310 .setAudioDevice(CallIntegration.AudioDevice.SPEAKER_PHONE);
1311 releaseProximityWakeLock();
1312 }
1313
1314 private void retry(final View view) {
1315 final Intent intent = getIntent();
1316 final Account account = extractAccount(intent);
1317 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1318 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1319 final String action = intent.getAction();
1320 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1321 this.rtpConnectionReference = null;
1322 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1323 CallIntegrationConnectionService.placeCall(xmppConnectionService, account, with, media);
1324 }
1325
1326 private void exit(final View view) {
1327 finish();
1328 }
1329
1330 private void recordVoiceMail(final View view) {
1331 final Intent intent = getIntent();
1332 final Account account = extractAccount(intent);
1333 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1334 final Conversation conversation =
1335 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1336 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1337 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1338 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1339 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1340 launchIntent.putExtra(
1341 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1342 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1343 startActivity(launchIntent);
1344 finish();
1345 }
1346
1347 private Contact getWith() {
1348 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1349 final Account account = id.account;
1350 return account.getRoster().getContact(id.with);
1351 }
1352
1353 private JingleRtpConnection requireRtpConnection() {
1354 final JingleRtpConnection connection =
1355 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1356 if (connection == null) {
1357 throw new IllegalStateException("No RTP connection found");
1358 }
1359 return connection;
1360 }
1361
1362 @Override
1363 public void onJingleRtpConnectionUpdate(
1364 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1365 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1366 if (END_CARD.contains(state)) {
1367 Log.d(Config.LOGTAG, "end card reached");
1368 releaseProximityWakeLock();
1369 runOnUiThread(
1370 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1371 }
1372 if (with.isBareJid()) {
1373 updateRtpSessionProposalState(account, with, state);
1374 return;
1375 }
1376 if (emptyReference(this.rtpConnectionReference)) {
1377 if (END_CARD.contains(state)) {
1378 Log.d(Config.LOGTAG, "not reinitializing session");
1379 return;
1380 }
1381 // this happens when going from proposed session to actual session
1382 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1383 return;
1384 }
1385 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1386 final boolean verified = requireRtpConnection().isVerified();
1387 final Set<Media> media = getMedia();
1388 lockOrientation(media);
1389 final ContentAddition contentAddition = getPendingContentAddition();
1390 final Contact contact = getWith();
1391 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1392 if (state == RtpEndUserState.ENDED) {
1393 finish();
1394 return;
1395 }
1396 runOnUiThread(
1397 () -> {
1398 updateStateDisplay(state, media, contentAddition);
1399 updateVerifiedShield(
1400 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1401 updateButtonConfiguration(state, media, contentAddition);
1402 updateVideoViews(state);
1403 updateIncomingCallScreen(state, contact);
1404 invalidateOptionsMenu();
1405 });
1406 if (END_CARD.contains(state)) {
1407 final JingleRtpConnection rtpConnection = requireRtpConnection();
1408 resetIntent(account, with, state, rtpConnection.getMedia());
1409 releaseVideoTracks(rtpConnection);
1410 this.rtpConnectionReference = null;
1411 }
1412 } else {
1413 Log.d(Config.LOGTAG, "received update for other rtp session");
1414 }
1415 }
1416
1417 @Override
1418 public void onAudioDeviceChanged(
1419 final CallIntegration.AudioDevice selectedAudioDevice,
1420 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
1421 Log.d(
1422 Config.LOGTAG,
1423 "onAudioDeviceChanged in activity: selected:"
1424 + selectedAudioDevice
1425 + ", available:"
1426 + availableAudioDevices);
1427 try {
1428 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1429 final Set<Media> media = getMedia();
1430 if (END_CARD.contains(endUserState)) {
1431 Log.d(
1432 Config.LOGTAG,
1433 "onAudioDeviceChanged() nothing to do because end card has been reached");
1434 } else {
1435 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1436 final CallIntegration callIntegration =
1437 requireRtpConnection().getCallIntegration();
1438 updateInCallButtonConfigurationSpeaker(
1439 callIntegration.getSelectedAudioDevice(),
1440 callIntegration.getAudioDevices().size());
1441 }
1442 Log.d(
1443 Config.LOGTAG,
1444 "put proximity wake lock into proper state after device update");
1445 putProximityWakeLockInProperState(selectedAudioDevice);
1446 }
1447 } catch (final IllegalStateException e) {
1448 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1449 }
1450 }
1451
1452 private void updateRtpSessionProposalState(
1453 final Account account, final Jid with, final RtpEndUserState state) {
1454 final Intent currentIntent = getIntent();
1455 final String withExtra =
1456 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1457 if (withExtra == null) {
1458 return;
1459 }
1460 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1461 runOnUiThread(
1462 () -> {
1463 updateVerifiedShield(false);
1464 updateStateDisplay(state);
1465 updateButtonConfiguration(state);
1466 updateIncomingCallScreen(state);
1467 invalidateOptionsMenu();
1468 });
1469 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1470 }
1471 }
1472
1473 private void resetIntent(final Bundle extras) {
1474 final Intent intent = new Intent(Intent.ACTION_VIEW);
1475 intent.putExtras(extras);
1476 setIntent(intent);
1477 }
1478
1479 private void resetIntent(
1480 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1481 final Intent intent = new Intent(Intent.ACTION_VIEW);
1482 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1483 if (RtpCapability.jmiSupport(account.getRoster().getContact(with))) {
1484 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1485 } else {
1486 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1487 }
1488 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1489 intent.putExtra(
1490 EXTRA_LAST_ACTION,
1491 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1492 setIntent(intent);
1493 }
1494
1495 private static boolean emptyReference(final WeakReference<?> weakReference) {
1496 return weakReference == null || weakReference.get() == null;
1497 }
1498}