1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.Activity;
9import android.app.PictureInPictureParams;
10import android.content.ActivityNotFoundException;
11import android.content.Context;
12import android.content.Intent;
13import android.content.pm.ActivityInfo;
14import android.content.pm.PackageManager;
15import android.os.Build;
16import android.os.Bundle;
17import android.os.Handler;
18import android.os.PowerManager;
19import android.os.SystemClock;
20import android.util.Log;
21import android.util.Rational;
22import android.view.KeyEvent;
23import android.view.Menu;
24import android.view.MenuItem;
25import android.view.View;
26import android.view.WindowManager;
27import android.widget.Toast;
28
29import androidx.annotation.NonNull;
30import androidx.annotation.Nullable;
31import androidx.annotation.RequiresApi;
32import androidx.annotation.StringRes;
33import androidx.databinding.DataBindingUtil;
34
35import com.google.common.base.Optional;
36import com.google.common.base.Preconditions;
37import com.google.common.base.Throwables;
38import com.google.common.collect.ImmutableList;
39import com.google.common.collect.ImmutableSet;
40import com.google.common.util.concurrent.FutureCallback;
41import com.google.common.util.concurrent.Futures;
42
43import org.webrtc.RendererCommon;
44import org.webrtc.SurfaceViewRenderer;
45import org.webrtc.VideoTrack;
46
47import java.lang.ref.WeakReference;
48import java.util.Arrays;
49import java.util.Collections;
50import java.util.List;
51import java.util.Set;
52
53import eu.siacs.conversations.Config;
54import eu.siacs.conversations.R;
55import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.services.XmppConnectionService;
61import eu.siacs.conversations.ui.util.AvatarWorkerTask;
62import eu.siacs.conversations.ui.util.MainThreadExecutor;
63import eu.siacs.conversations.ui.util.Rationals;
64import eu.siacs.conversations.utils.PermissionUtils;
65import eu.siacs.conversations.utils.TimeFrameUtils;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
69import eu.siacs.conversations.xmpp.jingle.ContentAddition;
70import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
71import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
72import eu.siacs.conversations.xmpp.jingle.Media;
73import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
74
75public class RtpSessionActivity extends XmppActivity
76 implements XmppConnectionService.OnJingleRtpConnectionUpdate,
77 eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
78
79 public static final String EXTRA_WITH = "with";
80 public static final String EXTRA_SESSION_ID = "session_id";
81 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
82 public static final String EXTRA_LAST_ACTION = "last_action";
83 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
84 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
85 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
86
87 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
88
89 private static final List<RtpEndUserState> END_CARD =
90 Arrays.asList(
91 RtpEndUserState.APPLICATION_ERROR,
92 RtpEndUserState.SECURITY_ERROR,
93 RtpEndUserState.DECLINED_OR_BUSY,
94 RtpEndUserState.CONNECTIVITY_ERROR,
95 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
96 RtpEndUserState.RETRACTED);
97 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON =
98 Arrays.asList(
99 RtpEndUserState.APPLICATION_ERROR,
100 RtpEndUserState.CONNECTIVITY_ERROR,
101 RtpEndUserState.SECURITY_ERROR);
102 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT =
103 Arrays.asList(
104 RtpEndUserState.CONNECTING,
105 RtpEndUserState.CONNECTED,
106 RtpEndUserState.RECONNECTING,
107 RtpEndUserState.INCOMING_CONTENT_ADD);
108 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED =
109 Arrays.asList(
110 RtpEndUserState.CONNECTED,
111 RtpEndUserState.RECONNECTING);
112 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER =
113 Arrays.asList(
114 RtpEndUserState.ACCEPTING_CALL,
115 RtpEndUserState.CONNECTING,
116 RtpEndUserState.RECONNECTING);
117 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
118 private static final int REQUEST_ACCEPT_CALL = 0x1111;
119 private static final int REQUEST_ACCEPT_CONTENT = 0x1112;
120 private static final int REQUEST_ADD_CONTENT = 0x1113;
121 private WeakReference<JingleRtpConnection> rtpConnectionReference;
122
123 private ActivityRtpSessionBinding binding;
124 private PowerManager.WakeLock mProximityWakeLock;
125
126 private final Handler mHandler = new Handler();
127 private final Runnable mTickExecutor =
128 new Runnable() {
129 @Override
130 public void run() {
131 updateCallDuration();
132 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
133 }
134 };
135
136 private static Set<Media> actionToMedia(final String action) {
137 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
138 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
139 } else {
140 return ImmutableSet.of(Media.AUDIO);
141 }
142 }
143
144 private static void addSink(
145 final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
146 try {
147 videoTrack.addSink(surfaceViewRenderer);
148 } catch (final IllegalStateException e) {
149 Log.e(
150 Config.LOGTAG,
151 "possible race condition on trying to display video track. ignoring",
152 e);
153 }
154 }
155
156 @Override
157 public void onCreate(Bundle savedInstanceState) {
158 super.onCreate(savedInstanceState);
159 getWindow()
160 .addFlags(
161 WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
162 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
163 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
164 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
165 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
166 setSupportActionBar(binding.toolbar);
167 }
168
169 @Override
170 public boolean onCreateOptionsMenu(final Menu menu) {
171 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
172 final MenuItem help = menu.findItem(R.id.action_help);
173 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
174 final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video);
175 help.setVisible(Config.HELP != null && isHelpButtonVisible());
176 gotoChat.setVisible(isSwitchToConversationVisible());
177 switchToVideo.setVisible(isSwitchToVideoVisible());
178 return super.onCreateOptionsMenu(menu);
179 }
180
181 @Override
182 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
183 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
184 if (xmppConnectionService != null) {
185 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
186 return true;
187 }
188 }
189 }
190 return super.onKeyDown(keyCode, event);
191 }
192
193 private boolean isHelpButtonVisible() {
194 try {
195 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
196 } catch (IllegalStateException e) {
197 final Intent intent = getIntent();
198 final String state =
199 intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
200 if (state != null) {
201 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
202 } else {
203 return false;
204 }
205 }
206 }
207
208 private boolean isSwitchToConversationVisible() {
209 final JingleRtpConnection connection =
210 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
211 return connection != null
212 && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
213 }
214
215 private boolean isSwitchToVideoVisible() {
216 final JingleRtpConnection connection =
217 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
218 if (connection == null) {
219 return false;
220 }
221 return connection.isSwitchToVideoAvailable();
222 }
223
224 private void switchToConversation() {
225 final Contact contact = getWith();
226 final Conversation conversation =
227 xmppConnectionService.findOrCreateConversation(
228 contact.getAccount(), contact.getJid(), false, true);
229 switchToConversation(conversation);
230 }
231
232 public boolean onOptionsItemSelected(final MenuItem item) {
233 switch (item.getItemId()) {
234 case R.id.action_help:
235 launchHelpInBrowser();
236 return true;
237 case R.id.action_goto_chat:
238 switchToConversation();
239 return true;
240 case R.id.action_switch_to_video:
241 requestPermissionAndSwitchToVideo();
242 return true;
243 }
244 return super.onOptionsItemSelected(item);
245 }
246
247 private void launchHelpInBrowser() {
248 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
249 try {
250 startActivity(intent);
251 } catch (final ActivityNotFoundException e) {
252 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG)
253 .show();
254 }
255 }
256
257 private void endCall(View view) {
258 endCall();
259 }
260
261 private void endCall() {
262 if (this.rtpConnectionReference == null) {
263 retractSessionProposal();
264 finish();
265 } else {
266 requireRtpConnection().endCall();
267 }
268 }
269
270 private void retractSessionProposal() {
271 final Intent intent = getIntent();
272 final String action = intent.getAction();
273 final Account account = extractAccount(intent);
274 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
275 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
276 if (!Intent.ACTION_VIEW.equals(action)
277 || state == null
278 || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
279 resetIntent(
280 account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
281 }
282 xmppConnectionService
283 .getJingleConnectionManager()
284 .retractSessionProposal(account, with.asBareJid());
285 }
286
287 private void rejectCall(View view) {
288 requireRtpConnection().rejectCall();
289 finish();
290 }
291
292 private void acceptCall(View view) {
293 requestPermissionsAndAcceptCall();
294 }
295
296 private void acceptContentAdd() {
297 try {
298 requireRtpConnection()
299 .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary);
300 } catch (final IllegalStateException e) {
301 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
302 }
303 }
304
305 private void requestPermissionAndSwitchToVideo() {
306 final List<String> permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO));
307 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) {
308 switchToVideo();
309 }
310 }
311
312 private void switchToVideo() {
313 try {
314 requireRtpConnection().addMedia(Media.VIDEO);
315 } catch (final IllegalStateException e) {
316 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
317 }
318 }
319
320 private void acceptContentAdd(final ContentAddition contentAddition) {
321 if (contentAddition == null || contentAddition.direction != ContentAddition.Direction.INCOMING) {
322 Log.d(Config.LOGTAG,"ignore press on content-accept button");
323 return;
324 }
325 requestPermissionAndAcceptContentAdd(contentAddition);
326 }
327
328 private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) {
329 final List<String> permissions = permissions(contentAddition.media());
330 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) {
331 requireRtpConnection().acceptContentAdd(contentAddition.summary);
332 }
333 }
334
335 private void rejectContentAdd(final View view) {
336 requireRtpConnection().rejectContentAdd();
337 }
338
339 private void requestPermissionsAndAcceptCall() {
340 final List<String> permissions = permissions(getMedia());
341 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
342 putScreenInCallMode();
343 checkRecorderAndAcceptCall();
344 }
345 }
346
347 private List<String> permissions(final Set<Media> media) {
348 final ImmutableList.Builder<String> permissions = ImmutableList.builder();
349 if (media.contains(Media.VIDEO)) {
350 permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO);
351 } else {
352 permissions.add(Manifest.permission.RECORD_AUDIO);
353 }
354 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
355 permissions.add(Manifest.permission.BLUETOOTH_CONNECT);
356 }
357 return permissions.build();
358 }
359
360 private void checkRecorderAndAcceptCall() {
361 checkMicrophoneAvailabilityAsync();
362 try {
363 requireRtpConnection().acceptCall();
364 } catch (final IllegalStateException e) {
365 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
366 }
367 }
368
369 private void checkMicrophoneAvailabilityAsync() {
370 new Thread(new MicrophoneAvailabilityCheck(this)).start();
371 }
372
373 private static class MicrophoneAvailabilityCheck implements Runnable {
374
375 private final WeakReference<Activity> activityReference;
376
377 private MicrophoneAvailabilityCheck(final Activity activity) {
378 this.activityReference = new WeakReference<>(activity);
379 }
380
381 @Override
382 public void run() {
383 final long start = SystemClock.elapsedRealtime();
384 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
385 final long stop = SystemClock.elapsedRealtime();
386 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
387 if (isMicrophoneAvailable) {
388 return;
389 }
390 final Activity activity = activityReference.get();
391 if (activity == null) {
392 return;
393 }
394 activity.runOnUiThread(
395 () ->
396 Toast.makeText(
397 activity,
398 R.string.microphone_unavailable,
399 Toast.LENGTH_LONG)
400 .show());
401 }
402 }
403
404 private void putScreenInCallMode() {
405 putScreenInCallMode(requireRtpConnection().getMedia());
406 }
407
408 private void putScreenInCallMode(final Set<Media> media) {
409 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
410 if (Media.audioOnly(media)) {
411 final JingleRtpConnection rtpConnection =
412 rtpConnectionReference != null ? rtpConnectionReference.get() : null;
413 final AppRTCAudioManager audioManager =
414 rtpConnection == null ? null : rtpConnection.getAudioManager();
415 if (audioManager == null
416 || audioManager.getSelectedAudioDevice()
417 == AppRTCAudioManager.AudioDevice.EARPIECE) {
418 acquireProximityWakeLock();
419 }
420 }
421 lockOrientation(media);
422 }
423
424 private void lockOrientation(final Set<Media> media) {
425 if (Media.audioOnly(media)) {
426 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
427 } else {
428 setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED);
429 }
430 }
431
432 @SuppressLint("WakelockTimeout")
433 private void acquireProximityWakeLock() {
434 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
435 if (powerManager == null) {
436 Log.e(Config.LOGTAG, "power manager not available");
437 return;
438 }
439 if (isFinishing()) {
440 Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing");
441 return;
442 }
443 if (this.mProximityWakeLock == null) {
444 this.mProximityWakeLock =
445 powerManager.newWakeLock(
446 PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
447 }
448 if (!this.mProximityWakeLock.isHeld()) {
449 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
450 this.mProximityWakeLock.acquire();
451 }
452 }
453
454 private void releaseProximityWakeLock() {
455 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
456 Log.d(Config.LOGTAG, "releasing proximity wake lock");
457 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
458 this.mProximityWakeLock = null;
459 }
460 }
461
462 private void putProximityWakeLockInProperState(
463 final AppRTCAudioManager.AudioDevice audioDevice) {
464 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
465 acquireProximityWakeLock();
466 } else {
467 releaseProximityWakeLock();
468 }
469 }
470
471 @Override
472 protected void refreshUiReal() {}
473
474 @Override
475 public void onNewIntent(final Intent intent) {
476 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
477 super.onNewIntent(intent);
478 setIntent(intent);
479 if (xmppConnectionService == null) {
480 Log.d(
481 Config.LOGTAG,
482 "RtpSessionActivity: background service wasn't bound in onNewIntent()");
483 return;
484 }
485 final Account account = extractAccount(intent);
486 final String action = intent.getAction();
487 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
488 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
489 if (sessionId != null) {
490 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
491 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
492 return;
493 }
494 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
495 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
496 requestPermissionsAndAcceptCall();
497 resetIntent(intent.getExtras());
498 }
499 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
500 proposeJingleRtpSession(account, with, actionToMedia(action));
501 setWith(account.getRoster().getContact(with), null);
502 } else {
503 throw new IllegalStateException("received onNewIntent without sessionId");
504 }
505 }
506
507 @Override
508 void onBackendConnected() {
509 final Intent intent = getIntent();
510 final String action = intent.getAction();
511 final Account account = extractAccount(intent);
512 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
513 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
514 if (sessionId != null) {
515 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
516 return;
517 }
518 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
519 Log.d(Config.LOGTAG, "intent action was accept");
520 requestPermissionsAndAcceptCall();
521 resetIntent(intent.getExtras());
522 }
523 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
524 proposeJingleRtpSession(account, with, actionToMedia(action));
525 setWith(account.getRoster().getContact(with), null);
526 } else if (Intent.ACTION_VIEW.equals(action)) {
527 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
528 final RtpEndUserState state =
529 extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
530 if (state != null) {
531 Log.d(Config.LOGTAG, "restored last state from intent extra");
532 updateButtonConfiguration(state);
533 updateVerifiedShield(false);
534 updateStateDisplay(state);
535 updateIncomingCallScreen(state);
536 invalidateOptionsMenu();
537 }
538 setWith(account.getRoster().getContact(with), state);
539 if (xmppConnectionService
540 .getJingleConnectionManager()
541 .fireJingleRtpConnectionStateUpdates()) {
542 return;
543 }
544 if (END_CARD.contains(state)
545 || xmppConnectionService
546 .getJingleConnectionManager()
547 .hasMatchingProposal(account, with)) {
548 return;
549 }
550 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
551 finish();
552 }
553 }
554
555 private void setWidth(final RtpEndUserState state) {
556 setWith(getWith(), state);
557 }
558
559 private void setWith(final Contact contact, final RtpEndUserState state) {
560 binding.with.setText(contact.getDisplayName());
561 if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL)
562 .contains(state)) {
563 binding.withJid.setText(contact.getJid().asBareJid().toEscapedString());
564 binding.withJid.setVisibility(View.VISIBLE);
565 } else {
566 binding.withJid.setVisibility(View.GONE);
567 }
568 }
569
570 private void proposeJingleRtpSession(
571 final Account account, final Jid with, final Set<Media> media) {
572 checkMicrophoneAvailabilityAsync();
573 if (with.isBareJid()) {
574 xmppConnectionService
575 .getJingleConnectionManager()
576 .proposeJingleRtpSession(account, with, media);
577 } else {
578 final String sessionId =
579 xmppConnectionService
580 .getJingleConnectionManager()
581 .initializeRtpSession(account, with, media);
582 initializeActivityWithRunningRtpSession(account, with, sessionId);
583 resetIntent(account, with, sessionId);
584 }
585 putScreenInCallMode(media);
586 }
587
588 @Override
589 public void onRequestPermissionsResult(
590 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
591 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
592 final PermissionUtils.PermissionResult permissionResult =
593 PermissionUtils.removeBluetoothConnect(permissions, grantResults);
594 if (PermissionUtils.allGranted(permissionResult.grantResults)) {
595 if (requestCode == REQUEST_ACCEPT_CALL) {
596 checkRecorderAndAcceptCall();
597 } else if (requestCode == REQUEST_ACCEPT_CONTENT) {
598 acceptContentAdd();
599 } else if (requestCode == REQUEST_ADD_CONTENT) {
600 switchToVideo();
601 }
602 } else {
603 @StringRes int res;
604 final String firstDenied =
605 getFirstDenied(permissionResult.grantResults, permissionResult.permissions);
606 if (firstDenied == null) {
607 return;
608 }
609 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
610 res = R.string.no_microphone_permission;
611 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
612 res = R.string.no_camera_permission;
613 } else {
614 throw new IllegalStateException("Invalid permission result request");
615 }
616 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT)
617 .show();
618 }
619 }
620
621 @Override
622 public void onStart() {
623 super.onStart();
624 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
625 this.binding.remoteVideo.setOnAspectRatioChanged(this);
626 }
627
628 @Override
629 public void onStop() {
630 mHandler.removeCallbacks(mTickExecutor);
631 binding.remoteVideo.release();
632 binding.remoteVideo.setOnAspectRatioChanged(null);
633 binding.localVideo.release();
634 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
635 final JingleRtpConnection jingleRtpConnection =
636 weakReference == null ? null : weakReference.get();
637 if (jingleRtpConnection != null) {
638 releaseVideoTracks(jingleRtpConnection);
639 }
640 releaseProximityWakeLock();
641 super.onStop();
642 }
643
644 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
645 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
646 if (remoteVideo.isPresent()) {
647 remoteVideo.get().removeSink(binding.remoteVideo);
648 }
649 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
650 if (localVideo.isPresent()) {
651 localVideo.get().removeSink(binding.localVideo);
652 }
653 }
654
655 @Override
656 public void onBackPressed() {
657 if (isConnected()) {
658 if (switchToPictureInPicture()) {
659 return;
660 }
661 } else {
662 endCall();
663 }
664 super.onBackPressed();
665 }
666
667 @Override
668 public void onUserLeaveHint() {
669 super.onUserLeaveHint();
670 if (switchToPictureInPicture()) {
671 return;
672 }
673 // TODO apparently this method is not getting called on Android 10 when using the task
674 // switcher
675 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
676 retractSessionProposal();
677 }
678 }
679
680 private boolean isConnected() {
681 final JingleRtpConnection connection =
682 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
683 final RtpEndUserState endUserState = connection == null ? null : connection.getEndUserState();
684 return STATES_CONSIDERED_CONNECTED.contains(endUserState) || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD;
685 }
686
687 private boolean switchToPictureInPicture() {
688 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
689 if (shouldBePictureInPicture()) {
690 startPictureInPicture();
691 return true;
692 }
693 }
694 return false;
695 }
696
697 @RequiresApi(api = Build.VERSION_CODES.O)
698 private void startPictureInPicture() {
699 try {
700 final Rational rational = this.binding.remoteVideo.getAspectRatio();
701 final Rational clippedRational = Rationals.clip(rational);
702 Log.d(
703 Config.LOGTAG,
704 "suggested rational " + rational + ". clipped to " + clippedRational);
705 enterPictureInPictureMode(
706 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
707 } catch (final IllegalStateException e) {
708 // this sometimes happens on Samsung phones (possibly when Knox is enabled)
709 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
710 }
711 }
712
713 @Override
714 public void onAspectRatioChanged(final Rational rational) {
715 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
716 final Rational clippedRational = Rationals.clip(rational);
717 Log.d(
718 Config.LOGTAG,
719 "suggested rational after aspect ratio change "
720 + rational
721 + ". clipped to "
722 + clippedRational);
723 setPictureInPictureParams(
724 new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build());
725 }
726 }
727
728 private boolean deviceSupportsPictureInPicture() {
729 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
730 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
731 } else {
732 return false;
733 }
734 }
735
736 private boolean shouldBePictureInPicture() {
737 try {
738 final JingleRtpConnection rtpConnection = requireRtpConnection();
739 return rtpConnection.getMedia().contains(Media.VIDEO)
740 && Arrays.asList(
741 RtpEndUserState.ACCEPTING_CALL,
742 RtpEndUserState.CONNECTING,
743 RtpEndUserState.CONNECTED)
744 .contains(rtpConnection.getEndUserState());
745 } catch (final IllegalStateException e) {
746 return false;
747 }
748 }
749
750 private boolean initializeActivityWithRunningRtpSession(
751 final Account account, Jid with, String sessionId) {
752 final WeakReference<JingleRtpConnection> reference =
753 xmppConnectionService
754 .getJingleConnectionManager()
755 .findJingleRtpConnection(account, with, sessionId);
756 if (reference == null || reference.get() == null) {
757 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession =
758 xmppConnectionService
759 .getJingleConnectionManager()
760 .getTerminalSessionState(with, sessionId);
761 if (terminatedRtpSession == null) {
762 throw new IllegalStateException(
763 "failed to initialize activity with running rtp session. session not found");
764 }
765 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
766 return true;
767 }
768 this.rtpConnectionReference = reference;
769 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
770 final boolean verified = requireRtpConnection().isVerified();
771 if (currentState == RtpEndUserState.ENDED) {
772 finish();
773 return true;
774 }
775 final Set<Media> media = getMedia();
776 final ContentAddition contentAddition = getPendingContentAddition();
777 if (currentState == RtpEndUserState.INCOMING_CALL) {
778 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
779 }
780 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(
781 requireRtpConnection().getState())) {
782 putScreenInCallMode();
783 }
784 setWidth(currentState);
785 updateVideoViews(currentState);
786 updateStateDisplay(currentState, media, contentAddition);
787 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
788 updateButtonConfiguration(currentState, media, contentAddition);
789 updateIncomingCallScreen(currentState);
790 invalidateOptionsMenu();
791 return false;
792 }
793
794 private void initializeWithTerminatedSessionState(
795 final Account account,
796 final Jid with,
797 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
798 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
799 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
800 finish();
801 return;
802 }
803 final RtpEndUserState state = terminatedRtpSession.state;
804 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
805 updateButtonConfiguration(state);
806 updateStateDisplay(state);
807 updateIncomingCallScreen(state);
808 updateCallDuration();
809 updateVerifiedShield(false);
810 invalidateOptionsMenu();
811 setWith(account.getRoster().getContact(with), state);
812 }
813
814 private void reInitializeActivityWithRunningRtpSession(
815 final Account account, Jid with, String sessionId) {
816 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
817 resetIntent(account, with, sessionId);
818 }
819
820 private void resetIntent(final Account account, final Jid with, final String sessionId) {
821 final Intent intent = new Intent(Intent.ACTION_VIEW);
822 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
823 intent.putExtra(EXTRA_WITH, with.toEscapedString());
824 intent.putExtra(EXTRA_SESSION_ID, sessionId);
825 setIntent(intent);
826 }
827
828 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
829 surfaceViewRenderer.setVisibility(View.VISIBLE);
830 try {
831 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
832 } catch (final IllegalStateException e) {
833 // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
834 }
835 surfaceViewRenderer.setEnableHardwareScaler(true);
836 }
837
838 private void updateStateDisplay(final RtpEndUserState state) {
839 updateStateDisplay(state, Collections.emptySet(), null);
840 }
841
842 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
843 switch (state) {
844 case INCOMING_CALL:
845 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
846 if (media.contains(Media.VIDEO)) {
847 setTitle(R.string.rtp_state_incoming_video_call);
848 } else {
849 setTitle(R.string.rtp_state_incoming_call);
850 }
851 break;
852 case INCOMING_CONTENT_ADD:
853 if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) {
854 setTitle(R.string.rtp_state_content_add_video);
855 } else {
856 setTitle(R.string.rtp_state_content_add);
857 }
858 break;
859 case CONNECTING:
860 setTitle(R.string.rtp_state_connecting);
861 break;
862 case CONNECTED:
863 setTitle(R.string.rtp_state_connected);
864 break;
865 case RECONNECTING:
866 setTitle(R.string.rtp_state_reconnecting);
867 break;
868 case ACCEPTING_CALL:
869 setTitle(R.string.rtp_state_accepting_call);
870 break;
871 case ENDING_CALL:
872 setTitle(R.string.rtp_state_ending_call);
873 break;
874 case FINDING_DEVICE:
875 setTitle(R.string.rtp_state_finding_device);
876 break;
877 case RINGING:
878 setTitle(R.string.rtp_state_ringing);
879 break;
880 case DECLINED_OR_BUSY:
881 setTitle(R.string.rtp_state_declined_or_busy);
882 break;
883 case CONNECTIVITY_ERROR:
884 setTitle(R.string.rtp_state_connectivity_error);
885 break;
886 case CONNECTIVITY_LOST_ERROR:
887 setTitle(R.string.rtp_state_connectivity_lost_error);
888 break;
889 case RETRACTED:
890 setTitle(R.string.rtp_state_retracted);
891 break;
892 case APPLICATION_ERROR:
893 setTitle(R.string.rtp_state_application_failure);
894 break;
895 case SECURITY_ERROR:
896 setTitle(R.string.rtp_state_security_error);
897 break;
898 case ENDED:
899 throw new IllegalStateException(
900 "Activity should have called finishAndReleaseWakeLock();");
901 default:
902 throw new IllegalStateException(
903 String.format("State %s has not been handled in UI", state));
904 }
905 }
906
907 private void updateVerifiedShield(final boolean verified) {
908 if (isPictureInPicture()) {
909 this.binding.verified.setVisibility(View.GONE);
910 return;
911 }
912 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
913 }
914
915 private void updateIncomingCallScreen(final RtpEndUserState state) {
916 updateIncomingCallScreen(state, null);
917 }
918
919 private void updateIncomingCallScreen(final RtpEndUserState state, final Contact contact) {
920 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
921 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
922 if (show) {
923 binding.contactPhoto.setVisibility(View.VISIBLE);
924 if (contact == null) {
925 AvatarWorkerTask.loadAvatar(
926 getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
927 } else {
928 AvatarWorkerTask.loadAvatar(
929 contact, binding.contactPhoto, R.dimen.publish_avatar_size);
930 }
931 } else {
932 binding.contactPhoto.setVisibility(View.GONE);
933 }
934 final Account account = contact == null ? getWith().getAccount() : contact.getAccount();
935 binding.usingAccount.setVisibility(View.VISIBLE);
936 binding.usingAccount.setText(
937 getString(
938 R.string.using_account,
939 account.getJid().asBareJid().toEscapedString()));
940 } else {
941 binding.usingAccount.setVisibility(View.GONE);
942 binding.contactPhoto.setVisibility(View.GONE);
943 }
944 }
945
946 private Set<Media> getMedia() {
947 return requireRtpConnection().getMedia();
948 }
949
950 public ContentAddition getPendingContentAddition() {
951 return requireRtpConnection().getPendingContentAddition();
952 }
953
954 private void updateButtonConfiguration(final RtpEndUserState state) {
955 updateButtonConfiguration(state, Collections.emptySet(), null);
956 }
957
958 @SuppressLint("RestrictedApi")
959 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media, final ContentAddition contentAddition) {
960 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
961 this.binding.rejectCall.setVisibility(View.INVISIBLE);
962 this.binding.endCall.setVisibility(View.INVISIBLE);
963 this.binding.acceptCall.setVisibility(View.INVISIBLE);
964 } else if (state == RtpEndUserState.INCOMING_CALL) {
965 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
966 this.binding.rejectCall.setOnClickListener(this::rejectCall);
967 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
968 this.binding.rejectCall.setVisibility(View.VISIBLE);
969 this.binding.endCall.setVisibility(View.INVISIBLE);
970 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
971 this.binding.acceptCall.setOnClickListener(this::acceptCall);
972 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
973 this.binding.acceptCall.setVisibility(View.VISIBLE);
974 } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) {
975 this.binding.rejectCall.setContentDescription(getString(R.string.reject_switch_to_video));
976 this.binding.rejectCall.setOnClickListener(this::rejectContentAdd);
977 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
978 this.binding.rejectCall.setVisibility(View.VISIBLE);
979 this.binding.endCall.setVisibility(View.INVISIBLE);
980 this.binding.acceptCall.setContentDescription(getString(R.string.accept));
981 this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition)));
982 this.binding.acceptCall.setImageResource(R.drawable.ic_baseline_check_24);
983 this.binding.acceptCall.setVisibility(View.VISIBLE);
984 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
985 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
986 this.binding.rejectCall.setOnClickListener(this::exit);
987 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
988 this.binding.rejectCall.setVisibility(View.VISIBLE);
989 this.binding.endCall.setVisibility(View.INVISIBLE);
990 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
991 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
992 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
993 this.binding.acceptCall.setVisibility(View.VISIBLE);
994 } else if (asList(
995 RtpEndUserState.CONNECTIVITY_ERROR,
996 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
997 RtpEndUserState.APPLICATION_ERROR,
998 RtpEndUserState.RETRACTED,
999 RtpEndUserState.SECURITY_ERROR)
1000 .contains(state)) {
1001 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
1002 this.binding.rejectCall.setOnClickListener(this::exit);
1003 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
1004 this.binding.rejectCall.setVisibility(View.VISIBLE);
1005 this.binding.endCall.setVisibility(View.INVISIBLE);
1006 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
1007 this.binding.acceptCall.setOnClickListener(this::retry);
1008 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
1009 this.binding.acceptCall.setVisibility(View.VISIBLE);
1010 } else {
1011 this.binding.rejectCall.setVisibility(View.INVISIBLE);
1012 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
1013 this.binding.endCall.setOnClickListener(this::endCall);
1014 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
1015 this.binding.endCall.setVisibility(View.VISIBLE);
1016 this.binding.acceptCall.setVisibility(View.INVISIBLE);
1017 }
1018 updateInCallButtonConfiguration(state, media);
1019 }
1020
1021 private boolean isPictureInPicture() {
1022 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
1023 return isInPictureInPictureMode();
1024 } else {
1025 return false;
1026 }
1027 }
1028
1029 private void updateInCallButtonConfiguration() {
1030 updateInCallButtonConfiguration(
1031 requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
1032 }
1033
1034 @SuppressLint("RestrictedApi")
1035 private void updateInCallButtonConfiguration(
1036 final RtpEndUserState state, final Set<Media> media) {
1037 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
1038 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
1039 if (media.contains(Media.VIDEO)) {
1040 final JingleRtpConnection rtpConnection = requireRtpConnection();
1041 updateInCallButtonConfigurationVideo(
1042 rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
1043 } else {
1044 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1045 updateInCallButtonConfigurationSpeaker(
1046 audioManager.getSelectedAudioDevice(),
1047 audioManager.getAudioDevices().size());
1048 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1049 }
1050 if (media.contains(Media.AUDIO)) {
1051 updateInCallButtonConfigurationMicrophone(
1052 requireRtpConnection().isMicrophoneEnabled());
1053 } else {
1054 this.binding.inCallActionLeft.setVisibility(View.GONE);
1055 }
1056 } else {
1057 this.binding.inCallActionLeft.setVisibility(View.GONE);
1058 this.binding.inCallActionRight.setVisibility(View.GONE);
1059 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1060 }
1061 }
1062
1063 @SuppressLint("RestrictedApi")
1064 private void updateInCallButtonConfigurationSpeaker(
1065 final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
1066 switch (selectedAudioDevice) {
1067 case EARPIECE:
1068 this.binding.inCallActionRight.setImageResource(
1069 R.drawable.ic_volume_off_black_24dp);
1070 if (numberOfChoices >= 2) {
1071 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
1072 } else {
1073 this.binding.inCallActionRight.setOnClickListener(null);
1074 this.binding.inCallActionRight.setClickable(false);
1075 }
1076 break;
1077 case WIRED_HEADSET:
1078 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
1079 this.binding.inCallActionRight.setOnClickListener(null);
1080 this.binding.inCallActionRight.setClickable(false);
1081 break;
1082 case SPEAKER_PHONE:
1083 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
1084 if (numberOfChoices >= 2) {
1085 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
1086 } else {
1087 this.binding.inCallActionRight.setOnClickListener(null);
1088 this.binding.inCallActionRight.setClickable(false);
1089 }
1090 break;
1091 case BLUETOOTH:
1092 this.binding.inCallActionRight.setImageResource(
1093 R.drawable.ic_bluetooth_audio_black_24dp);
1094 this.binding.inCallActionRight.setOnClickListener(null);
1095 this.binding.inCallActionRight.setClickable(false);
1096 break;
1097 }
1098 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1099 }
1100
1101 @SuppressLint("RestrictedApi")
1102 private void updateInCallButtonConfigurationVideo(
1103 final boolean videoEnabled, final boolean isCameraSwitchable) {
1104 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
1105 if (isCameraSwitchable) {
1106 this.binding.inCallActionFarRight.setImageResource(
1107 R.drawable.ic_flip_camera_android_black_24dp);
1108 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
1109 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
1110 } else {
1111 this.binding.inCallActionFarRight.setVisibility(View.GONE);
1112 }
1113 if (videoEnabled) {
1114 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
1115 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
1116 } else {
1117 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
1118 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
1119 }
1120 }
1121
1122 private void switchCamera(final View view) {
1123 Futures.addCallback(
1124 requireRtpConnection().switchCamera(),
1125 new FutureCallback<Boolean>() {
1126 @Override
1127 public void onSuccess(@Nullable Boolean isFrontCamera) {
1128 binding.localVideo.setMirror(isFrontCamera);
1129 }
1130
1131 @Override
1132 public void onFailure(@NonNull final Throwable throwable) {
1133 Log.d(
1134 Config.LOGTAG,
1135 "could not switch camera",
1136 Throwables.getRootCause(throwable));
1137 Toast.makeText(
1138 RtpSessionActivity.this,
1139 R.string.could_not_switch_camera,
1140 Toast.LENGTH_LONG)
1141 .show();
1142 }
1143 },
1144 MainThreadExecutor.getInstance());
1145 }
1146
1147 private void enableVideo(View view) {
1148 try {
1149 requireRtpConnection().setVideoEnabled(true);
1150 } catch (final IllegalStateException e) {
1151 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
1152 return;
1153 }
1154 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
1155 }
1156
1157 private void disableVideo(View view) {
1158 final JingleRtpConnection rtpConnection = requireRtpConnection();
1159 final ContentAddition pending = rtpConnection.getPendingContentAddition();
1160 if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) {
1161 rtpConnection.retractContentAdd();
1162 return;
1163 }
1164 requireRtpConnection().setVideoEnabled(false);
1165 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
1166 }
1167
1168 @SuppressLint("RestrictedApi")
1169 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
1170 if (microphoneEnabled) {
1171 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
1172 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
1173 } else {
1174 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
1175 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
1176 }
1177 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
1178 }
1179
1180 private void updateCallDuration() {
1181 final JingleRtpConnection connection =
1182 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1183 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
1184 this.binding.duration.setVisibility(View.GONE);
1185 return;
1186 }
1187 if (connection.zeroDuration()) {
1188 this.binding.duration.setVisibility(View.GONE);
1189 } else {
1190 this.binding.duration.setText(
1191 TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1192 this.binding.duration.setVisibility(View.VISIBLE);
1193 }
1194 }
1195
1196 private void updateVideoViews(final RtpEndUserState state) {
1197 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1198 binding.localVideo.setVisibility(View.GONE);
1199 binding.localVideo.release();
1200 binding.remoteVideoWrapper.setVisibility(View.GONE);
1201 binding.remoteVideo.release();
1202 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1203 if (isPictureInPicture()) {
1204 binding.appBarLayout.setVisibility(View.GONE);
1205 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1206 if (Arrays.asList(
1207 RtpEndUserState.APPLICATION_ERROR,
1208 RtpEndUserState.CONNECTIVITY_ERROR,
1209 RtpEndUserState.SECURITY_ERROR)
1210 .contains(state)) {
1211 binding.pipWarning.setVisibility(View.VISIBLE);
1212 binding.pipWaiting.setVisibility(View.GONE);
1213 } else {
1214 binding.pipWarning.setVisibility(View.GONE);
1215 binding.pipWaiting.setVisibility(View.GONE);
1216 }
1217 } else {
1218 binding.appBarLayout.setVisibility(View.VISIBLE);
1219 binding.pipPlaceholder.setVisibility(View.GONE);
1220 }
1221 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1222 return;
1223 }
1224 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1225 binding.localVideo.setVisibility(View.GONE);
1226 binding.remoteVideoWrapper.setVisibility(View.GONE);
1227 binding.appBarLayout.setVisibility(View.GONE);
1228 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1229 binding.pipWarning.setVisibility(View.GONE);
1230 binding.pipWaiting.setVisibility(View.VISIBLE);
1231 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1232 return;
1233 }
1234 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1235 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1236 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1237 // paint local view over remote view
1238 binding.localVideo.setZOrderMediaOverlay(true);
1239 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1240 addSink(localVideoTrack.get(), binding.localVideo);
1241 } else {
1242 binding.localVideo.setVisibility(View.GONE);
1243 }
1244 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1245 if (remoteVideoTrack.isPresent()) {
1246 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1247 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1248 binding.remoteVideo.setScalingType(
1249 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1250 RendererCommon.ScalingType.SCALE_ASPECT_FIT);
1251 if (state == RtpEndUserState.CONNECTED) {
1252 binding.appBarLayout.setVisibility(View.GONE);
1253 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1254 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1255 } else {
1256 binding.appBarLayout.setVisibility(View.VISIBLE);
1257 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1258 binding.remoteVideoWrapper.setVisibility(View.GONE);
1259 }
1260 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1261 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1262 } else {
1263 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1264 }
1265 } else {
1266 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1267 binding.remoteVideoWrapper.setVisibility(View.GONE);
1268 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1269 }
1270 }
1271
1272 private Optional<VideoTrack> getLocalVideoTrack() {
1273 final JingleRtpConnection connection =
1274 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1275 if (connection == null) {
1276 return Optional.absent();
1277 }
1278 return connection.getLocalVideoTrack();
1279 }
1280
1281 private Optional<VideoTrack> getRemoteVideoTrack() {
1282 final JingleRtpConnection connection =
1283 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1284 if (connection == null) {
1285 return Optional.absent();
1286 }
1287 return connection.getRemoteVideoTrack();
1288 }
1289
1290 private void disableMicrophone(View view) {
1291 final JingleRtpConnection rtpConnection = requireRtpConnection();
1292 if (rtpConnection.setMicrophoneEnabled(false)) {
1293 updateInCallButtonConfiguration();
1294 }
1295 }
1296
1297 private void enableMicrophone(View view) {
1298 final JingleRtpConnection rtpConnection = requireRtpConnection();
1299 if (rtpConnection.setMicrophoneEnabled(true)) {
1300 updateInCallButtonConfiguration();
1301 }
1302 }
1303
1304 private void switchToEarpiece(View view) {
1305 requireRtpConnection()
1306 .getAudioManager()
1307 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1308 acquireProximityWakeLock();
1309 }
1310
1311 private void switchToSpeaker(View view) {
1312 requireRtpConnection()
1313 .getAudioManager()
1314 .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1315 releaseProximityWakeLock();
1316 }
1317
1318 private void retry(View view) {
1319 final Intent intent = getIntent();
1320 final Account account = extractAccount(intent);
1321 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1322 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1323 final String action = intent.getAction();
1324 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1325 this.rtpConnectionReference = null;
1326 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1327 proposeJingleRtpSession(account, with, media);
1328 }
1329
1330 private void exit(final View view) {
1331 finish();
1332 }
1333
1334 private void recordVoiceMail(final View view) {
1335 final Intent intent = getIntent();
1336 final Account account = extractAccount(intent);
1337 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1338 final Conversation conversation =
1339 xmppConnectionService.findOrCreateConversation(account, with, false, true);
1340 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1341 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1342 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1343 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1344 launchIntent.putExtra(
1345 ConversationsActivity.EXTRA_POST_INIT_ACTION,
1346 ConversationsActivity.POST_ACTION_RECORD_VOICE);
1347 startActivity(launchIntent);
1348 finish();
1349 }
1350
1351 private Contact getWith() {
1352 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1353 final Account account = id.account;
1354 return account.getRoster().getContact(id.with);
1355 }
1356
1357 private JingleRtpConnection requireRtpConnection() {
1358 final JingleRtpConnection connection =
1359 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1360 if (connection == null) {
1361 throw new IllegalStateException("No RTP connection found");
1362 }
1363 return connection;
1364 }
1365
1366 @Override
1367 public void onJingleRtpConnectionUpdate(
1368 Account account, Jid with, final String sessionId, RtpEndUserState state) {
1369 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1370 if (END_CARD.contains(state)) {
1371 Log.d(Config.LOGTAG, "end card reached");
1372 releaseProximityWakeLock();
1373 runOnUiThread(
1374 () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1375 }
1376 if (with.isBareJid()) {
1377 updateRtpSessionProposalState(account, with, state);
1378 return;
1379 }
1380 if (emptyReference(this.rtpConnectionReference)) {
1381 if (END_CARD.contains(state)) {
1382 Log.d(Config.LOGTAG, "not reinitializing session");
1383 return;
1384 }
1385 // this happens when going from proposed session to actual session
1386 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1387 return;
1388 }
1389 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1390 final boolean verified = requireRtpConnection().isVerified();
1391 final Set<Media> media = getMedia();
1392 lockOrientation(media);
1393 final ContentAddition contentAddition = getPendingContentAddition();
1394 final Contact contact = getWith();
1395 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1396 if (state == RtpEndUserState.ENDED) {
1397 finish();
1398 return;
1399 }
1400 runOnUiThread(
1401 () -> {
1402 updateStateDisplay(state, media, contentAddition);
1403 updateVerifiedShield(
1404 verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1405 updateButtonConfiguration(state, media, contentAddition);
1406 updateVideoViews(state);
1407 updateIncomingCallScreen(state, contact);
1408 invalidateOptionsMenu();
1409 });
1410 if (END_CARD.contains(state)) {
1411 final JingleRtpConnection rtpConnection = requireRtpConnection();
1412 resetIntent(account, with, state, rtpConnection.getMedia());
1413 releaseVideoTracks(rtpConnection);
1414 this.rtpConnectionReference = null;
1415 }
1416 } else {
1417 Log.d(Config.LOGTAG, "received update for other rtp session");
1418 }
1419 }
1420
1421 @Override
1422 public void onAudioDeviceChanged(
1423 final AppRTCAudioManager.AudioDevice selectedAudioDevice,
1424 final Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1425 Log.d(
1426 Config.LOGTAG,
1427 "onAudioDeviceChanged in activity: selected:"
1428 + selectedAudioDevice
1429 + ", available:"
1430 + availableAudioDevices);
1431 try {
1432 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1433 final Set<Media> media = getMedia();
1434 if (END_CARD.contains(endUserState)) {
1435 Log.d(
1436 Config.LOGTAG,
1437 "onAudioDeviceChanged() nothing to do because end card has been reached");
1438 } else {
1439 if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) {
1440 final AppRTCAudioManager audioManager =
1441 requireRtpConnection().getAudioManager();
1442 updateInCallButtonConfigurationSpeaker(
1443 audioManager.getSelectedAudioDevice(),
1444 audioManager.getAudioDevices().size());
1445 }
1446 Log.d(
1447 Config.LOGTAG,
1448 "put proximity wake lock into proper state after device update");
1449 putProximityWakeLockInProperState(selectedAudioDevice);
1450 }
1451 } catch (final IllegalStateException e) {
1452 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1453 }
1454 }
1455
1456 private void updateRtpSessionProposalState(
1457 final Account account, final Jid with, final RtpEndUserState state) {
1458 final Intent currentIntent = getIntent();
1459 final String withExtra =
1460 currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1461 if (withExtra == null) {
1462 return;
1463 }
1464 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1465 runOnUiThread(
1466 () -> {
1467 updateVerifiedShield(false);
1468 updateStateDisplay(state);
1469 updateButtonConfiguration(state);
1470 updateIncomingCallScreen(state);
1471 invalidateOptionsMenu();
1472 });
1473 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1474 }
1475 }
1476
1477 private void resetIntent(final Bundle extras) {
1478 final Intent intent = new Intent(Intent.ACTION_VIEW);
1479 intent.putExtras(extras);
1480 setIntent(intent);
1481 }
1482
1483 private void resetIntent(
1484 final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1485 final Intent intent = new Intent(Intent.ACTION_VIEW);
1486 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1487 if (account.getRoster()
1488 .getContact(with)
1489 .getPresences()
1490 .anySupport(Namespace.JINGLE_MESSAGE)) {
1491 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1492 } else {
1493 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1494 }
1495 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1496 intent.putExtra(
1497 EXTRA_LAST_ACTION,
1498 media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1499 setIntent(intent);
1500 }
1501
1502 private static boolean emptyReference(final WeakReference<?> weakReference) {
1503 return weakReference == null || weakReference.get() == null;
1504 }
1505}