1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.os.Build;
16import android.os.Bundle;
17import android.os.Handler;
18import android.os.PowerManager;
19import android.os.SystemClock;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import org.webrtc.RendererCommon;
44import org.webrtc.SurfaceViewRenderer;
45import org.webrtc.VideoTrack;
46
47import java.lang.ref.WeakReference;
48import java.util.Arrays;
49import java.util.Collections;
50import java.util.List;
51import java.util.Set;
52
53import eu.siacs.conversations.Config;
54import eu.siacs.conversations.R;
55import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.services.XmppConnectionService;
61import eu.siacs.conversations.ui.util.AvatarWorkerTask;
62import eu.siacs.conversations.ui.util.MainThreadExecutor;
63import eu.siacs.conversations.ui.util.Rationals;
64import eu.siacs.conversations.utils.PermissionUtils;
65import eu.siacs.conversations.utils.TimeFrameUtils;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
69import eu.siacs.conversations.xmpp.jingle.ContentAddition;
70import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
71import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
72import eu.siacs.conversations.xmpp.jingle.Media;
73import eu.siacs.conversations.xmpp.jingle.RtpCapability;
74import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
75
76public class RtpSessionActivity extends XmppActivity
77 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
78 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
79
80 public static final String EXTRA_WITH = "with";
81 public static final String EXTRA_SESSION_ID = "session_id";
82 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
83 public static final String EXTRA_LAST_ACTION = "last_action";
84 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
85 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
86 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
87
88 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
89
90 private static final List<RtpEndUserState> END_CARD =
91 Arrays.asList(
92 RtpEndUserState.APPLICATION_ERROR,
93 RtpEndUserState.SECURITY_ERROR,
94 RtpEndUserState.DECLINED_OR_BUSY,
95 RtpEndUserState.CONNECTIVITY_ERROR,
96 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
97 RtpEndUserState.RETRACTED);
98 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
99 Arrays.asList(
100 RtpEndUserState.APPLICATION_ERROR,
101 RtpEndUserState.CONNECTIVITY_ERROR,
102 RtpEndUserState.SECURITY_ERROR);
103 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
104 Arrays.asList(
105 RtpEndUserState.CONNECTING,
106 RtpEndUserState.CONNECTED,
107 RtpEndUserState.RECONNECTING,
108 RtpEndUserState.INCOMING_CONTENT_ADD);
109 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
110 Arrays.asList(
111 RtpEndUserState.CONNECTED,
112 RtpEndUserState.RECONNECTING);
113 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
114 Arrays.asList(
115 RtpEndUserState.ACCEPTING_CALL,
116 RtpEndUserState.CONNECTING,
117 RtpEndUserState.RECONNECTING);
118 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
119 private static final int REQUEST_ACCEPT_CALL = 0x1111;
120 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
121 private static final int REQUEST_ADD_CONTENT = 0x1113;
122 private WeakReference<JingleRtpConnection> rtpConnectionReference;
123
124 private ActivityRtpSessionBinding binding;
125 private PowerManager.WakeLock mProximityWakeLock;
126
127 private final Handler mHandler = new Handler();
128 private final Runnable mTickExecutor =
129 new Runnable() {
130 @Override
131 public void run() {
132 updateCallDuration();
133 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
134 }
135 };
136
137 private static Set<Media> actionToMedia(final String action) {
138 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
139 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
140 } else {
141 return ImmutableSet.of(Media.AUDIO);
142 }
143 }
144
145 private static void addSink(
146 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
147 try {
148 videoTrack.addSink(surfaceViewRenderer);
149 } catch (final IllegalStateException e) {
150 Log.e(
151 Config.LOGTAG,
152 "possible race condition on trying to display video track. ignoring",
153 e);
154 }
155 }
156
157 @Override
158 public void onCreate(Bundle savedInstanceState) {
159 super.onCreate(savedInstanceState);
160 getWindow()
161 .addFlags(
162 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
163 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
164 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
165 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
166 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
167 setSupportActionBar(binding.toolbar);
168 }
169
170 @Override
171 public boolean onCreateOptionsMenu(final Menu menu) {
172 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
173 final MenuItem help = menu.findItem(R.id.action_help);
174 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
175 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
176 help.setVisible(Config.HELP != null && isHelpButtonVisible());
177 gotoChat.setVisible(isSwitchToConversationVisible());
178 switchToVideo.setVisible(isSwitchToVideoVisible());
179 return super.onCreateOptionsMenu(menu);
180 }
181
182 @Override
183 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
184 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
185 if (xmppConnectionService != null) {
186 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
187 return true;
188 }
189 }
190 }
191 return super.onKeyDown(keyCode, event);
192 }
193
194 private boolean isHelpButtonVisible() {
195 try {
196 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
197 } catch (IllegalStateException e) {
198 final Intent intent = getIntent();
199 final String state =
200 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
201 if (state != null) {
202 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
203 } else {
204 return false;
205 }
206 }
207 }
208
209 private boolean isSwitchToConversationVisible() {
210 final JingleRtpConnection connection =
211 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
212 return connection != null
213 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
214 }
215
216 private boolean isSwitchToVideoVisible() {
217 final JingleRtpConnection connection =
218 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
219 if (connection == null) {
220 return false;
221 }
222 return connection.isSwitchToVideoAvailable();
223 }
224
225 private void switchToConversation() {
226 final Contact contact = getWith();
227 final Conversation conversation =
228 xmppConnectionService.findOrCreateConversation(
229 contact.getAccount(), contact.getJid(), false, true);
230 switchToConversation(conversation);
231 }
232
233 public boolean onOptionsItemSelected(final MenuItem item) {
234 switch (item.getItemId()) {
235 case R.id.action_help:
236 launchHelpInBrowser();
237 return true;
238 case R.id.action_goto_chat:
239 switchToConversation();
240 return true;
241 case R.id.action_switch_to_video:
242 requestPermissionAndSwitchToVideo();
243 return true;
244 }
245 return super.onOptionsItemSelected(item);
246 }
247
248 private void launchHelpInBrowser() {
249 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
250 try {
251 startActivity(intent);
252 } catch (final ActivityNotFoundException e) {
253 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
254 .show();
255 }
256 }
257
258 private void endCall(View view) {
259 endCall();
260 }
261
262 private void endCall() {
263 if (this.rtpConnectionReference == null) {
264 retractSessionProposal();
265 finish();
266 } else {
267 requireRtpConnection().endCall();
268 }
269 }
270
271 private void retractSessionProposal() {
272 final Intent intent = getIntent();
273 final String action = intent.getAction();
274 final Account account = extractAccount(intent);
275 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
276 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
277 if (!Intent.ACTION_VIEW.equals(action)
278 || state == null
279 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
280 resetIntent(
281 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
282 }
283 xmppConnectionService
284 .getJingleConnectionManager()
285 .retractSessionProposal(account, with.asBareJid());
286 }
287
288 private void rejectCall(View view) {
289 requireRtpConnection().rejectCall();
290 finish();
291 }
292
293 private void acceptCall(View view) {
294 requestPermissionsAndAcceptCall();
295 }
296
297 private void acceptContentAdd() {
298 try {
299 requireRtpConnection()
300 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
301 } catch (final IllegalStateException e) {
302 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
303 }
304 }
305
306 private void requestPermissionAndSwitchToVideo() {
307 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
308 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
309 switchToVideo();
310 }
311 }
312
313 private void switchToVideo() {
314 try {
315 requireRtpConnection().addMedia(Media.VIDEO);
316 } catch (final IllegalStateException e) {
317 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
318 }
319 }
320
321 private void acceptContentAdd(final ContentAddition contentAddition) {
322 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
323 Log.d(Config.LOGTAG,"ignore press on content-accept button");
324 return;
325 }
326 requestPermissionAndAcceptContentAdd(contentAddition);
327 }
328
329 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
330 final List<String> permissions = permissions(contentAddition.media());
331 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
332 requireRtpConnection().acceptContentAdd(contentAddition.summary);
333 }
334 }
335
336 private void rejectContentAdd(final View view) {
337 requireRtpConnection().rejectContentAdd();
338 }
339
340 private void requestPermissionsAndAcceptCall() {
341 final List<String> permissions = permissions(getMedia());
342 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
343 putScreenInCallMode();
344 checkRecorderAndAcceptCall();
345 }
346 }
347
348 private List<String> permissions(final Set<Media> media) {
349 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
350 if (media.contains(Media.VIDEO)) {
351 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
352 } else {
353 permissions.add(Manifest.permission.RECORD_AUDIO);
354 }
355 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
356 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
357 }
358 return permissions.build();
359 }
360
361 private void checkRecorderAndAcceptCall() {
362 checkMicrophoneAvailabilityAsync();
363 try {
364 requireRtpConnection().acceptCall();
365 } catch (final IllegalStateException e) {
366 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
367 }
368 }
369
370 private void checkMicrophoneAvailabilityAsync() {
371 new Thread(new MicrophoneAvailabilityCheck(this)).start();
372 }
373
374 private static class MicrophoneAvailabilityCheck implements Runnable {
375
376 private final WeakReference<Activity> activityReference;
377
378 private MicrophoneAvailabilityCheck(final Activity activity) {
379 this.activityReference = new WeakReference<>(activity);
380 }
381
382 @Override
383 public void run() {
384 final long start = SystemClock.elapsedRealtime();
385 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
386 final long stop = SystemClock.elapsedRealtime();
387 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
388 if (isMicrophoneAvailable) {
389 return;
390 }
391 final Activity activity = activityReference.get();
392 if (activity == null) {
393 return;
394 }
395 activity.runOnUiThread(
396 () ->
397 Toast.makeText(
398 activity,
399 R.string.microphone_unavailable,
400 Toast.LENGTH_LONG)
401 .show());
402 }
403 }
404
405 private void putScreenInCallMode() {
406 putScreenInCallMode(requireRtpConnection().getMedia());
407 }
408
409 private void putScreenInCallMode(final Set<Media> media) {
410 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
411 if (Media.audioOnly(media)) {
412 final JingleRtpConnection rtpConnection =
413 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
414 final AppRTCAudioManager audioManager =
415 rtpConnection == null ? null : rtpConnection.getAudioManager();
416 if (audioManager == null
417 || audioManager.getSelectedAudioDevice()
418 == AppRTCAudioManager.AudioDevice.EARPIECE) {
419 acquireProximityWakeLock();
420 }
421 }
422 lockOrientation(media);
423 }
424
425 private void lockOrientation(final Set<Media> media) {
426 if (Media.audioOnly(media)) {
427 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
428 } else {
429 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
430 }
431 }
432
433 @SuppressLint("WakelockTimeout")
434 private void acquireProximityWakeLock() {
435 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
436 if (powerManager == null) {
437 Log.e(Config.LOGTAG, "power manager not available");
438 return;
439 }
440 if (isFinishing()) {
441 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
442 return;
443 }
444 if (this.mProximityWakeLock == null) {
445 this.mProximityWakeLock =
446 powerManager.newWakeLock(
447 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
448 }
449 if (!this.mProximityWakeLock.isHeld()) {
450 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
451 this.mProximityWakeLock.acquire();
452 }
453 }
454
455 private void releaseProximityWakeLock() {
456 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
457 Log.d(Config.LOGTAG, "releasing proximity wake lock");
458 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
459 this.mProximityWakeLock = null;
460 }
461 }
462
463 private void putProximityWakeLockInProperState(
464 final AppRTCAudioManager.AudioDevice audioDevice) {
465 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
466 acquireProximityWakeLock();
467 } else {
468 releaseProximityWakeLock();
469 }
470 }
471
472 @Override
473 protected void refreshUiReal() {}
474
475 @Override
476 public void onNewIntent(final Intent intent) {
477 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
478 super.onNewIntent(intent);
479 setIntent(intent);
480 if (xmppConnectionService == null) {
481 Log.d(
482 Config.LOGTAG,
483 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
484 return;
485 }
486 final Account account = extractAccount(intent);
487 final String action = intent.getAction();
488 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
489 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
490 if (sessionId != null) {
491 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
492 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
493 return;
494 }
495 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
496 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
497 requestPermissionsAndAcceptCall();
498 resetIntent(intent.getExtras());
499 }
500 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
501 proposeJingleRtpSession(account, with, actionToMedia(action));
502 setWith(account.getRoster().getContact(with), null);
503 } else {
504 throw new IllegalStateException("received onNewIntent without sessionId");
505 }
506 }
507
508 @Override
509 void onBackendConnected() {
510 final Intent intent = getIntent();
511 final String action = intent.getAction();
512 final Account account = extractAccount(intent);
513 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
514 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
515 if (sessionId != null) {
516 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
517 return;
518 }
519 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
520 Log.d(Config.LOGTAG, "intent action was accept");
521 requestPermissionsAndAcceptCall();
522 resetIntent(intent.getExtras());
523 }
524 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
525 proposeJingleRtpSession(account, with, actionToMedia(action));
526 setWith(account.getRoster().getContact(with), null);
527 } else if (Intent.ACTION_VIEW.equals(action)) {
528 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
529 final RtpEndUserState state =
530 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
531 if (state != null) {
532 Log.d(Config.LOGTAG, "restored last state from intent extra");
533 updateButtonConfiguration(state);
534 updateVerifiedShield(false);
535 updateStateDisplay(state);
536 updateIncomingCallScreen(state);
537 invalidateOptionsMenu();
538 }
539 setWith(account.getRoster().getContact(with), state);
540 if (xmppConnectionService
541 .getJingleConnectionManager()
542 .fireJingleRtpConnectionStateUpdates()) {
543 return;
544 }
545 if (END_CARD.contains(state)
546 || xmppConnectionService
547 .getJingleConnectionManager()
548 .hasMatchingProposal(account, with)) {
549 return;
550 }
551 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
552 finish();
553 }
554 }
555
556 private void setWidth(final RtpEndUserState state) {
557 setWith(getWith(), state);
558 }
559
560 private void setWith(final Contact contact, final RtpEndUserState state) {
561 binding.with.setText(contact.getDisplayName());
562 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
563 .contains(state)) {
564 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
565 binding.withJid.setVisibility(View.VISIBLE);
566 } else {
567 binding.withJid.setVisibility(View.GONE);
568 }
569 }
570
571 private void proposeJingleRtpSession(
572 final Account account, final Jid with, final Set<Media> media) {
573 checkMicrophoneAvailabilityAsync();
574 if (with.isBareJid()) {
575 xmppConnectionService
576 .getJingleConnectionManager()
577 .proposeJingleRtpSession(account, with, media);
578 } else {
579 final String sessionId =
580 xmppConnectionService
581 .getJingleConnectionManager()
582 .initializeRtpSession(account, with, media);
583 initializeActivityWithRunningRtpSession(account, with, sessionId);
584 resetIntent(account, with, sessionId);
585 }
586 putScreenInCallMode(media);
587 }
588
589 @Override
590 public void onRequestPermissionsResult(
591 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
592 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
593 final PermissionUtils.PermissionResult permissionResult =
594 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
595 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
596 if (requestCode == REQUEST_ACCEPT_CALL) {
597 checkRecorderAndAcceptCall();
598 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
599 acceptContentAdd();
600 } else if (requestCode == REQUEST_ADD_CONTENT) {
601 switchToVideo();
602 }
603 } else {
604 @StringRes int res;
605 final String firstDenied =
606 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
607 if (firstDenied == null) {
608 return;
609 }
610 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
611 res = R.string.no_microphone_permission;
612 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
613 res = R.string.no_camera_permission;
614 } else {
615 throw new IllegalStateException("Invalid permission result request");
616 }
617 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
618 .show();
619 }
620 }
621
622 @Override
623 public void onStart() {
624 super.onStart();
625 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
626 this.binding.remoteVideo.setOnAspectRatioChanged(this);
627 }
628
629 @Override
630 public void onStop() {
631 mHandler.removeCallbacks(mTickExecutor);
632 binding.remoteVideo.release();
633 binding.remoteVideo.setOnAspectRatioChanged(null);
634 binding.localVideo.release();
635 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
636 final JingleRtpConnection jingleRtpConnection =
637 weakReference == null ? null : weakReference.get();
638 if (jingleRtpConnection != null) {
639 releaseVideoTracks(jingleRtpConnection);
640 }
641 releaseProximityWakeLock();
642 super.onStop();
643 }
644
645 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
646 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
647 if (remoteVideo.isPresent()) {
648 remoteVideo.get().removeSink(binding.remoteVideo);
649 }
650 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
651 if (localVideo.isPresent()) {
652 localVideo.get().removeSink(binding.localVideo);
653 }
654 }
655
656 @Override
657 public void onBackPressed() {
658 if (isConnected()) {
659 if (switchToPictureInPicture()) {
660 return;
661 }
662 } else {
663 endCall();
664 }
665 super.onBackPressed();
666 }
667
668 @Override
669 public void onUserLeaveHint() {
670 super.onUserLeaveHint();
671 if (switchToPictureInPicture()) {
672 return;
673 }
674 // TODO apparently this method is not getting called on Android 10 when using the task
675 // switcher
676 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
677 retractSessionProposal();
678 }
679 }
680
681 private boolean isConnected() {
682 final JingleRtpConnection connection =
683 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
684 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
685 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
686 }
687
688 private boolean switchToPictureInPicture() {
689 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
690 if (shouldBePictureInPicture()) {
691 startPictureInPicture();
692 return true;
693 }
694 }
695 return false;
696 }
697
698 @RequiresApi(api = Build.VERSION_CODES.O)
699 private void startPictureInPicture() {
700 try {
701 final Rational rational = this.binding.remoteVideo.getAspectRatio();
702 final Rational clippedRational = Rationals.clip(rational);
703 Log.d(
704 Config.LOGTAG,
705 "suggested rational " + rational + ". clipped to " + clippedRational);
706 enterPictureInPictureMode(
707 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
708 } catch (final IllegalStateException e) {
709 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
710 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
711 }
712 }
713
714 @Override
715 public void onAspectRatioChanged(final Rational rational) {
716 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
717 final Rational clippedRational = Rationals.clip(rational);
718 Log.d(
719 Config.LOGTAG,
720 "suggested rational after aspect ratio change "
721 + rational
722 + ". clipped to "
723 + clippedRational);
724 setPictureInPictureParams(
725 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
726 }
727 }
728
729 private boolean deviceSupportsPictureInPicture() {
730 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
731 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
732 } else {
733 return false;
734 }
735 }
736
737 private boolean shouldBePictureInPicture() {
738 try {
739 final JingleRtpConnection rtpConnection = requireRtpConnection();
740 return rtpConnection.getMedia().contains(Media.VIDEO)
741 && Arrays.asList(
742 RtpEndUserState.ACCEPTING_CALL,
743 RtpEndUserState.CONNECTING,
744 RtpEndUserState.CONNECTED)
745 .contains(rtpConnection.getEndUserState());
746 } catch (final IllegalStateException e) {
747 return false;
748 }
749 }
750
751 private boolean initializeActivityWithRunningRtpSession(
752 final Account account, Jid with, String sessionId) {
753 final WeakReference<JingleRtpConnection> reference =
754 xmppConnectionService
755 .getJingleConnectionManager()
756 .findJingleRtpConnection(account, with, sessionId);
757 if (reference == null || reference.get() == null) {
758 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
759 xmppConnectionService
760 .getJingleConnectionManager()
761 .getTerminalSessionState(with, sessionId);
762 if (terminatedRtpSession == null) {
763 throw new IllegalStateException(
764 "failed to initialize activity with running rtp session. session not found");
765 }
766 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
767 return true;
768 }
769 this.rtpConnectionReference = reference;
770 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
771 final boolean verified = requireRtpConnection().isVerified();
772 if (currentState == RtpEndUserState.ENDED) {
773 finish();
774 return true;
775 }
776 final Set<Media> media = getMedia();
777 final ContentAddition contentAddition = getPendingContentAddition();
778 if (currentState == RtpEndUserState.INCOMING_CALL) {
779 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
780 }
781 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
782 requireRtpConnection().getState())) {
783 putScreenInCallMode();
784 }
785 setWidth(currentState);
786 updateVideoViews(currentState);
787 updateStateDisplay(currentState, media, contentAddition);
788 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
789 updateButtonConfiguration(currentState, media, contentAddition);
790 updateIncomingCallScreen(currentState);
791 invalidateOptionsMenu();
792 return false;
793 }
794
795 private void initializeWithTerminatedSessionState(
796 final Account account,
797 final Jid with,
798 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
799 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
800 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
801 finish();
802 return;
803 }
804 final RtpEndUserState state = terminatedRtpSession.state;
805 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
806 updateButtonConfiguration(state);
807 updateStateDisplay(state);
808 updateIncomingCallScreen(state);
809 updateCallDuration();
810 updateVerifiedShield(false);
811 invalidateOptionsMenu();
812 setWith(account.getRoster().getContact(with), state);
813 }
814
815 private void reInitializeActivityWithRunningRtpSession(
816 final Account account, Jid with, String sessionId) {
817 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
818 resetIntent(account, with, sessionId);
819 }
820
821 private void resetIntent(final Account account, final Jid with, final String sessionId) {
822 final Intent intent = new Intent(Intent.ACTION_VIEW);
823 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
824 intent.putExtra(EXTRA_WITH, with.toEscapedString());
825 intent.putExtra(EXTRA_SESSION_ID, sessionId);
826 setIntent(intent);
827 }
828
829 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
830 surfaceViewRenderer.setVisibility(View.VISIBLE);
831 try {
832 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
833 } catch (final IllegalStateException e) {
834 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
835 }
836 surfaceViewRenderer.setEnableHardwareScaler(true);
837 }
838
839 private void updateStateDisplay(final RtpEndUserState state) {
840 updateStateDisplay(state, Collections.emptySet(), null);
841 }
842
843 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
844 switch (state) {
845 case INCOMING_CALL:
846 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
847 if (media.contains(Media.VIDEO)) {
848 setTitle(R.string.rtp_state_incoming_video_call);
849 } else {
850 setTitle(R.string.rtp_state_incoming_call);
851 }
852 break;
853 case INCOMING_CONTENT_ADD:
854 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
855 setTitle(R.string.rtp_state_content_add_video);
856 } else {
857 setTitle(R.string.rtp_state_content_add);
858 }
859 break;
860 case CONNECTING:
861 setTitle(R.string.rtp_state_connecting);
862 break;
863 case CONNECTED:
864 setTitle(R.string.rtp_state_connected);
865 break;
866 case RECONNECTING:
867 setTitle(R.string.rtp_state_reconnecting);
868 break;
869 case ACCEPTING_CALL:
870 setTitle(R.string.rtp_state_accepting_call);
871 break;
872 case ENDING_CALL:
873 setTitle(R.string.rtp_state_ending_call);
874 break;
875 case FINDING_DEVICE:
876 setTitle(R.string.rtp_state_finding_device);
877 break;
878 case RINGING:
879 setTitle(R.string.rtp_state_ringing);
880 break;
881 case DECLINED_OR_BUSY:
882 setTitle(R.string.rtp_state_declined_or_busy);
883 break;
884 case CONNECTIVITY_ERROR:
885 setTitle(R.string.rtp_state_connectivity_error);
886 break;
887 case CONNECTIVITY_LOST_ERROR:
888 setTitle(R.string.rtp_state_connectivity_lost_error);
889 break;
890 case RETRACTED:
891 setTitle(R.string.rtp_state_retracted);
892 break;
893 case APPLICATION_ERROR:
894 setTitle(R.string.rtp_state_application_failure);
895 break;
896 case SECURITY_ERROR:
897 setTitle(R.string.rtp_state_security_error);
898 break;
899 case ENDED:
900 throw new IllegalStateException(
901 "Activity should have called finishAndReleaseWakeLock();");
902 default:
903 throw new IllegalStateException(
904 String.format("State %s has not been handled in UI", state));
905 }
906 }
907
908 private void updateVerifiedShield(final boolean verified) {
909 if (isPictureInPicture()) {
910 this.binding.verified.setVisibility(View.GONE);
911 return;
912 }
913 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
914 }
915
916 private void updateIncomingCallScreen(final RtpEndUserState state) {
917 updateIncomingCallScreen(state, null);
918 }
919
920 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
921 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
922 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
923 if (show) {
924 binding.contactPhoto.setVisibility(View.VISIBLE);
925 if (contact == null) {
926 AvatarWorkerTask.loadAvatar(
927 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
928 } else {
929 AvatarWorkerTask.loadAvatar(
930 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
931 }
932 } else {
933 binding.contactPhoto.setVisibility(View.GONE);
934 }
935 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
936 binding.usingAccount.setVisibility(View.VISIBLE);
937 binding.usingAccount.setText(
938 getString(
939 R.string.using_account,
940 account.getJid().asBareJid().toEscapedString()));
941 } else {
942 binding.usingAccount.setVisibility(View.GONE);
943 binding.contactPhoto.setVisibility(View.GONE);
944 }
945 }
946
947 private Set<Media> getMedia() {
948 return requireRtpConnection().getMedia();
949 }
950
951 public ContentAddition getPendingContentAddition() {
952 return requireRtpConnection().getPendingContentAddition();
953 }
954
955 private void updateButtonConfiguration(final RtpEndUserState state) {
956 updateButtonConfiguration(state, Collections.emptySet(), null);
957 }
958
959 @SuppressLint("RestrictedApi")
960 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
961 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
962 this.binding.rejectCall.setVisibility(View.INVISIBLE);
963 this.binding.endCall.setVisibility(View.INVISIBLE);
964 this.binding.acceptCall.setVisibility(View.INVISIBLE);
965 } else if (state == RtpEndUserState.INCOMING_CALL) {
966 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
967 this.binding.rejectCall.setOnClickListener(this::rejectCall);
968 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
969 this.binding.rejectCall.setVisibility(View.VISIBLE);
970 this.binding.endCall.setVisibility(View.INVISIBLE);
971 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
972 this.binding.acceptCall.setOnClickListener(this::acceptCall);
973 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
974 this.binding.acceptCall.setVisibility(View.VISIBLE);
975 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
976 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
977 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
978 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
979 this.binding.rejectCall.setVisibility(View.VISIBLE);
980 this.binding.endCall.setVisibility(View.INVISIBLE);
981 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
982 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
983 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
984 this.binding.acceptCall.setVisibility(View.VISIBLE);
985 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
986 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
987 this.binding.rejectCall.setOnClickListener(this::exit);
988 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
989 this.binding.rejectCall.setVisibility(View.VISIBLE);
990 this.binding.endCall.setVisibility(View.INVISIBLE);
991 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
992 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
993 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
994 this.binding.acceptCall.setVisibility(View.VISIBLE);
995 } else if (asList(
996 RtpEndUserState.CONNECTIVITY_ERROR,
997 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
998 RtpEndUserState.APPLICATION_ERROR,
999 RtpEndUserState.RETRACTED,
1000 RtpEndUserState.SECURITY_ERROR)
1001 .contains(state)) {
1002 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1003 this.binding.rejectCall.setOnClickListener(this::exit);
1004 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1005 this.binding.rejectCall.setVisibility(View.VISIBLE);
1006 this.binding.endCall.setVisibility(View.INVISIBLE);
1007 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1008 this.binding.acceptCall.setOnClickListener(this::retry);
1009 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1010 this.binding.acceptCall.setVisibility(View.VISIBLE);
1011 } else {
1012 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1013 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1014 this.binding.endCall.setOnClickListener(this::endCall);
1015 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1016 this.binding.endCall.setVisibility(View.VISIBLE);
1017 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1018 }
1019 updateInCallButtonConfiguration(state, media);
1020 }
1021
1022 private boolean isPictureInPicture() {
1023 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1024 return isInPictureInPictureMode();
1025 } else {
1026 return false;
1027 }
1028 }
1029
1030 private void updateInCallButtonConfiguration() {
1031 updateInCallButtonConfiguration(
1032 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1033 }
1034
1035 @SuppressLint("RestrictedApi")
1036 private void updateInCallButtonConfiguration(
1037 final RtpEndUserState state, final Set<Media> media) {
1038 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1039 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1040 if (media.contains(Media.VIDEO)) {
1041 final JingleRtpConnection rtpConnection = requireRtpConnection();
1042 updateInCallButtonConfigurationVideo(
1043 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1044 } else {
1045 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1046 updateInCallButtonConfigurationSpeaker(
1047 audioManager.getSelectedAudioDevice(),
1048 audioManager.getAudioDevices().size());
1049 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1050 }
1051 if (media.contains(Media.AUDIO)) {
1052 updateInCallButtonConfigurationMicrophone(
1053 requireRtpConnection().isMicrophoneEnabled());
1054 } else {
1055 this.binding.inCallActionLeft.setVisibility(View.GONE);
1056 }
1057 } else {
1058 this.binding.inCallActionLeft.setVisibility(View.GONE);
1059 this.binding.inCallActionRight.setVisibility(View.GONE);
1060 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1061 }
1062 }
1063
1064 @SuppressLint("RestrictedApi")
1065 private void updateInCallButtonConfigurationSpeaker(
1066 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1067 switch (selectedAudioDevice) {
1068 case EARPIECE:
1069 this.binding.inCallActionRight.setImageResource(
1070 R.drawable.ic_volume_off_black_24dp);
1071 if (numberOfChoices >= 2) {
1072 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1073 } else {
1074 this.binding.inCallActionRight.setOnClickListener(null);
1075 this.binding.inCallActionRight.setClickable(false);
1076 }
1077 break;
1078 case WIRED_HEADSET:
1079 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1080 this.binding.inCallActionRight.setOnClickListener(null);
1081 this.binding.inCallActionRight.setClickable(false);
1082 break;
1083 case SPEAKER_PHONE:
1084 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1085 if (numberOfChoices >= 2) {
1086 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1087 } else {
1088 this.binding.inCallActionRight.setOnClickListener(null);
1089 this.binding.inCallActionRight.setClickable(false);
1090 }
1091 break;
1092 case BLUETOOTH:
1093 this.binding.inCallActionRight.setImageResource(
1094 R.drawable.ic_bluetooth_audio_black_24dp);
1095 this.binding.inCallActionRight.setOnClickListener(null);
1096 this.binding.inCallActionRight.setClickable(false);
1097 break;
1098 }
1099 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1100 }
1101
1102 @SuppressLint("RestrictedApi")
1103 private void updateInCallButtonConfigurationVideo(
1104 final boolean videoEnabled, final boolean isCameraSwitchable) {
1105 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1106 if (isCameraSwitchable) {
1107 this.binding.inCallActionFarRight.setImageResource(
1108 R.drawable.ic_flip_camera_android_black_24dp);
1109 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1110 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1111 } else {
1112 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1113 }
1114 if (videoEnabled) {
1115 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1116 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1117 } else {
1118 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1119 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1120 }
1121 }
1122
1123 private void switchCamera(final View view) {
1124 Futures.addCallback(
1125 requireRtpConnection().switchCamera(),
1126 new FutureCallback<Boolean>() {
1127 @Override
1128 public void onSuccess(@Nullable Boolean isFrontCamera) {
1129 binding.localVideo.setMirror(isFrontCamera);
1130 }
1131
1132 @Override
1133 public void onFailure(@NonNull final Throwable throwable) {
1134 Log.d(
1135 Config.LOGTAG,
1136 "could not switch camera",
1137 Throwables.getRootCause(throwable));
1138 Toast.makeText(
1139 RtpSessionActivity.this,
1140 R.string.could_not_switch_camera,
1141 Toast.LENGTH_LONG)
1142 .show();
1143 }
1144 },
1145 MainThreadExecutor.getInstance());
1146 }
1147
1148 private void enableVideo(View view) {
1149 try {
1150 requireRtpConnection().setVideoEnabled(true);
1151 } catch (final IllegalStateException e) {
1152 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1153 return;
1154 }
1155 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1156 }
1157
1158 private void disableVideo(View view) {
1159 final JingleRtpConnection rtpConnection = requireRtpConnection();
1160 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1161 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1162 rtpConnection.retractContentAdd();
1163 return;
1164 }
1165 requireRtpConnection().setVideoEnabled(false);
1166 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1167 }
1168
1169 @SuppressLint("RestrictedApi")
1170 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1171 if (microphoneEnabled) {
1172 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1173 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1174 } else {
1175 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1176 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1177 }
1178 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1179 }
1180
1181 private void updateCallDuration() {
1182 final JingleRtpConnection connection =
1183 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1184 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1185 this.binding.duration.setVisibility(View.GONE);
1186 return;
1187 }
1188 if (connection.zeroDuration()) {
1189 this.binding.duration.setVisibility(View.GONE);
1190 } else {
1191 this.binding.duration.setText(
1192 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1193 this.binding.duration.setVisibility(View.VISIBLE);
1194 }
1195 }
1196
1197 private void updateVideoViews(final RtpEndUserState state) {
1198 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1199 binding.localVideo.setVisibility(View.GONE);
1200 binding.localVideo.release();
1201 binding.remoteVideoWrapper.setVisibility(View.GONE);
1202 binding.remoteVideo.release();
1203 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1204 if (isPictureInPicture()) {
1205 binding.appBarLayout.setVisibility(View.GONE);
1206 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1207 if (Arrays.asList(
1208 RtpEndUserState.APPLICATION_ERROR,
1209 RtpEndUserState.CONNECTIVITY_ERROR,
1210 RtpEndUserState.SECURITY_ERROR)
1211 .contains(state)) {
1212 binding.pipWarning.setVisibility(View.VISIBLE);
1213 binding.pipWaiting.setVisibility(View.GONE);
1214 } else {
1215 binding.pipWarning.setVisibility(View.GONE);
1216 binding.pipWaiting.setVisibility(View.GONE);
1217 }
1218 } else {
1219 binding.appBarLayout.setVisibility(View.VISIBLE);
1220 binding.pipPlaceholder.setVisibility(View.GONE);
1221 }
1222 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1223 return;
1224 }
1225 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1226 binding.localVideo.setVisibility(View.GONE);
1227 binding.remoteVideoWrapper.setVisibility(View.GONE);
1228 binding.appBarLayout.setVisibility(View.GONE);
1229 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1230 binding.pipWarning.setVisibility(View.GONE);
1231 binding.pipWaiting.setVisibility(View.VISIBLE);
1232 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1233 return;
1234 }
1235 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1236 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1237 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1238 // paint local view over remote view
1239 binding.localVideo.setZOrderMediaOverlay(true);
1240 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1241 addSink(localVideoTrack.get(), binding.localVideo);
1242 } else {
1243 binding.localVideo.setVisibility(View.GONE);
1244 }
1245 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1246 if (remoteVideoTrack.isPresent()) {
1247 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1248 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1249 binding.remoteVideo.setScalingType(
1250 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1251 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1252 if (state == RtpEndUserState.CONNECTED) {
1253 binding.appBarLayout.setVisibility(View.GONE);
1254 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1255 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1256 } else {
1257 binding.appBarLayout.setVisibility(View.VISIBLE);
1258 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1259 binding.remoteVideoWrapper.setVisibility(View.GONE);
1260 }
1261 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1262 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1263 } else {
1264 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1265 }
1266 } else {
1267 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1268 binding.remoteVideoWrapper.setVisibility(View.GONE);
1269 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1270 }
1271 }
1272
1273 private Optional<VideoTrack> getLocalVideoTrack() {
1274 final JingleRtpConnection connection =
1275 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1276 if (connection == null) {
1277 return Optional.absent();
1278 }
1279 return connection.getLocalVideoTrack();
1280 }
1281
1282 private Optional<VideoTrack> getRemoteVideoTrack() {
1283 final JingleRtpConnection connection =
1284 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1285 if (connection == null) {
1286 return Optional.absent();
1287 }
1288 return connection.getRemoteVideoTrack();
1289 }
1290
1291 private void disableMicrophone(View view) {
1292 final JingleRtpConnection rtpConnection = requireRtpConnection();
1293 if (rtpConnection.setMicrophoneEnabled(false)) {
1294 updateInCallButtonConfiguration();
1295 }
1296 }
1297
1298 private void enableMicrophone(View view) {
1299 final JingleRtpConnection rtpConnection = requireRtpConnection();
1300 if (rtpConnection.setMicrophoneEnabled(true)) {
1301 updateInCallButtonConfiguration();
1302 }
1303 }
1304
1305 private void switchToEarpiece(View view) {
1306 requireRtpConnection()
1307 .getAudioManager()
1308 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1309 acquireProximityWakeLock();
1310 }
1311
1312 private void switchToSpeaker(View view) {
1313 requireRtpConnection()
1314 .getAudioManager()
1315 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1316 releaseProximityWakeLock();
1317 }
1318
1319 private void retry(View view) {
1320 final Intent intent = getIntent();
1321 final Account account = extractAccount(intent);
1322 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1323 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1324 final String action = intent.getAction();
1325 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1326 this.rtpConnectionReference = null;
1327 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1328 proposeJingleRtpSession(account, with, media);
1329 }
1330
1331 private void exit(final View view) {
1332 finish();
1333 }
1334
1335 private void recordVoiceMail(final View view) {
1336 final Intent intent = getIntent();
1337 final Account account = extractAccount(intent);
1338 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1339 final Conversation conversation =
1340 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1341 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1342 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1343 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1344 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1345 launchIntent.putExtra(
1346 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1347 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1348 startActivity(launchIntent);
1349 finish();
1350 }
1351
1352 private Contact getWith() {
1353 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1354 final Account account = id.account;
1355 return account.getRoster().getContact(id.with);
1356 }
1357
1358 private JingleRtpConnection requireRtpConnection() {
1359 final JingleRtpConnection connection =
1360 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1361 if (connection == null) {
1362 throw new IllegalStateException("No RTP connection found");
1363 }
1364 return connection;
1365 }
1366
1367 @Override
1368 public void onJingleRtpConnectionUpdate(
1369 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1370 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1371 if (END_CARD.contains(state)) {
1372 Log.d(Config.LOGTAG, "end card reached");
1373 releaseProximityWakeLock();
1374 runOnUiThread(
1375 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1376 }
1377 if (with.isBareJid()) {
1378 updateRtpSessionProposalState(account, with, state);
1379 return;
1380 }
1381 if (emptyReference(this.rtpConnectionReference)) {
1382 if (END_CARD.contains(state)) {
1383 Log.d(Config.LOGTAG, "not reinitializing session");
1384 return;
1385 }
1386 // this happens when going from proposed session to actual session
1387 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1388 return;
1389 }
1390 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1391 final boolean verified = requireRtpConnection().isVerified();
1392 final Set<Media> media = getMedia();
1393 lockOrientation(media);
1394 final ContentAddition contentAddition = getPendingContentAddition();
1395 final Contact contact = getWith();
1396 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1397 if (state == RtpEndUserState.ENDED) {
1398 finish();
1399 return;
1400 }
1401 runOnUiThread(
1402 () -> {
1403 updateStateDisplay(state, media, contentAddition);
1404 updateVerifiedShield(
1405 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1406 updateButtonConfiguration(state, media, contentAddition);
1407 updateVideoViews(state);
1408 updateIncomingCallScreen(state, contact);
1409 invalidateOptionsMenu();
1410 });
1411 if (END_CARD.contains(state)) {
1412 final JingleRtpConnection rtpConnection = requireRtpConnection();
1413 resetIntent(account, with, state, rtpConnection.getMedia());
1414 releaseVideoTracks(rtpConnection);
1415 this.rtpConnectionReference = null;
1416 }
1417 } else {
1418 Log.d(Config.LOGTAG, "received update for other rtp session");
1419 }
1420 }
1421
1422 @Override
1423 public void onAudioDeviceChanged(
1424 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1425 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1426 Log.d(
1427 Config.LOGTAG,
1428 "onAudioDeviceChanged in activity: selected:"
1429 + selectedAudioDevice
1430 + ", available:"
1431 + availableAudioDevices);
1432 try {
1433 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1434 final Set<Media> media = getMedia();
1435 if (END_CARD.contains(endUserState)) {
1436 Log.d(
1437 Config.LOGTAG,
1438 "onAudioDeviceChanged() nothing to do because end card has been reached");
1439 } else {
1440 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1441 final AppRTCAudioManager audioManager =
1442 requireRtpConnection().getAudioManager();
1443 updateInCallButtonConfigurationSpeaker(
1444 audioManager.getSelectedAudioDevice(),
1445 audioManager.getAudioDevices().size());
1446 }
1447 Log.d(
1448 Config.LOGTAG,
1449 "put proximity wake lock into proper state after device update");
1450 putProximityWakeLockInProperState(selectedAudioDevice);
1451 }
1452 } catch (final IllegalStateException e) {
1453 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1454 }
1455 }
1456
1457 private void updateRtpSessionProposalState(
1458 final Account account, final Jid with, final RtpEndUserState state) {
1459 final Intent currentIntent = getIntent();
1460 final String withExtra =
1461 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1462 if (withExtra == null) {
1463 return;
1464 }
1465 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1466 runOnUiThread(
1467 () -> {
1468 updateVerifiedShield(false);
1469 updateStateDisplay(state);
1470 updateButtonConfiguration(state);
1471 updateIncomingCallScreen(state);
1472 invalidateOptionsMenu();
1473 });
1474 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1475 }
1476 }
1477
1478 private void resetIntent(final Bundle extras) {
1479 final Intent intent = new Intent(Intent.ACTION_VIEW);
1480 intent.putExtras(extras);
1481 setIntent(intent);
1482 }
1483
1484 private void resetIntent(
1485 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1486 final Intent intent = new Intent(Intent.ACTION_VIEW);
1487 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1488 if (RtpCapability.jmiSupport(account.getRoster()
1489 .getContact(with))) {
1490 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1491 } else {
1492 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1493 }
1494 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1495 intent.putExtra(
1496 EXTRA_LAST_ACTION,
1497 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1498 setIntent(intent);
1499 }
1500
1501 private static boolean emptyReference(final WeakReference<?> weakReference) {
1502 return weakReference == null || weakReference.get() == null;
1503 }
1504}