1package eu.siacs.conversations.ui;
2
3import android.Manifest;
4import android.annotation.SuppressLint;
5import android.app.PictureInPictureParams;
6import android.content.ActivityNotFoundException;
7import android.content.Context;
8import android.content.Intent;
9import android.content.pm.PackageManager;
10import android.databinding.DataBindingUtil;
11import android.os.Build;
12import android.os.Bundle;
13import android.os.Handler;
14import android.os.PowerManager;
15import android.os.SystemClock;
16import android.support.annotation.NonNull;
17import android.support.annotation.RequiresApi;
18import android.support.annotation.StringRes;
19import android.util.Log;
20import android.util.Rational;
21import android.view.Menu;
22import android.view.MenuItem;
23import android.view.View;
24import android.view.WindowManager;
25import android.widget.Toast;
26
27import com.google.common.base.Optional;
28import com.google.common.base.Preconditions;
29import com.google.common.base.Throwables;
30import com.google.common.collect.ImmutableList;
31import com.google.common.collect.ImmutableSet;
32import com.google.common.util.concurrent.FutureCallback;
33import com.google.common.util.concurrent.Futures;
34
35import org.checkerframework.checker.nullness.compatqual.NullableDecl;
36import org.webrtc.SurfaceViewRenderer;
37import org.webrtc.VideoTrack;
38
39import java.lang.ref.WeakReference;
40import java.util.Arrays;
41import java.util.Collections;
42import java.util.List;
43import java.util.Set;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.R;
47import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.services.XmppConnectionService;
53import eu.siacs.conversations.ui.util.AvatarWorkerTask;
54import eu.siacs.conversations.ui.util.MainThreadExecutor;
55import eu.siacs.conversations.utils.PermissionUtils;
56import eu.siacs.conversations.utils.TimeFrameUtils;
57import eu.siacs.conversations.xml.Namespace;
58import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
59import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
60import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
61import eu.siacs.conversations.xmpp.jingle.Media;
62import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
63import eu.siacs.conversations.xmpp.Jid;
64
65import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
66import static java.util.Arrays.asList;
67
68public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate {
69
70 public static final String EXTRA_WITH = "with";
71 public static final String EXTRA_SESSION_ID = "session_id";
72 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
73 public static final String EXTRA_LAST_ACTION = "last_action";
74 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
75 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
76 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
77
78 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
79
80 private static final List<RtpEndUserState> END_CARD = Arrays.asList(
81 RtpEndUserState.APPLICATION_ERROR,
82 RtpEndUserState.DECLINED_OR_BUSY,
83 RtpEndUserState.CONNECTIVITY_ERROR,
84 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
85 RtpEndUserState.RETRACTED
86 );
87 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON = Arrays.asList(
88 RtpEndUserState.APPLICATION_ERROR,
89 RtpEndUserState.CONNECTIVITY_ERROR
90 );
91 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
92 private static final int REQUEST_ACCEPT_CALL = 0x1111;
93 private WeakReference<JingleRtpConnection> rtpConnectionReference;
94
95 private ActivityRtpSessionBinding binding;
96 private PowerManager.WakeLock mProximityWakeLock;
97
98 private Handler mHandler = new Handler();
99 private Runnable mTickExecutor = new Runnable() {
100 @Override
101 public void run() {
102 updateCallDuration();
103 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
104 }
105 };
106
107 private static Set<Media> actionToMedia(final String action) {
108 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
109 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
110 } else {
111 return ImmutableSet.of(Media.AUDIO);
112 }
113 }
114
115 private static void addSink(final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
116 try {
117 videoTrack.addSink(surfaceViewRenderer);
118 } catch (final IllegalStateException e) {
119 Log.e(Config.LOGTAG, "possible race condition on trying to display video track. ignoring", e);
120 }
121 }
122
123 @Override
124 public void onCreate(Bundle savedInstanceState) {
125 super.onCreate(savedInstanceState);
126 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
127 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
128 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
129 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
130 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
131 setSupportActionBar(binding.toolbar);
132 }
133
134 @Override
135 public boolean onCreateOptionsMenu(final Menu menu) {
136 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
137 final MenuItem help = menu.findItem(R.id.action_help);
138 help.setVisible(isHelpButtonVisible());
139 return super.onCreateOptionsMenu(menu);
140 }
141
142 private boolean isHelpButtonVisible() {
143 try {
144 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
145 } catch (IllegalStateException e) {
146 final Intent intent = getIntent();
147 final String state = intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
148 if (state != null) {
149 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
150 } else {
151 return false;
152 }
153 }
154 }
155
156 public boolean onOptionsItemSelected(final MenuItem item) {
157 if (item.getItemId() == R.id.action_help) {
158 launchHelpInBrowser();
159 return true;
160 }
161 return super.onOptionsItemSelected(item);
162 }
163
164 private void launchHelpInBrowser() {
165 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
166 try {
167 startActivity(intent);
168 } catch (final ActivityNotFoundException e) {
169 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG).show();
170 }
171 }
172
173 private void endCall(View view) {
174 endCall();
175 }
176
177 private void endCall() {
178 if (this.rtpConnectionReference == null) {
179 retractSessionProposal();
180 finish();
181 } else {
182 requireRtpConnection().endCall();
183 }
184 }
185
186 private void retractSessionProposal() {
187 final Intent intent = getIntent();
188 final String action = intent.getAction();
189 final Account account = extractAccount(intent);
190 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
191 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
192 if (!Intent.ACTION_VIEW.equals(action) || state == null || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
193 resetIntent(account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
194 }
195 xmppConnectionService.getJingleConnectionManager().retractSessionProposal(account, with.asBareJid());
196 }
197
198 private void rejectCall(View view) {
199 requireRtpConnection().rejectCall();
200 finish();
201 }
202
203 private void acceptCall(View view) {
204 requestPermissionsAndAcceptCall();
205 }
206
207 private void requestPermissionsAndAcceptCall() {
208 final List<String> permissions;
209 if (getMedia().contains(Media.VIDEO)) {
210 permissions = ImmutableList.of(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO);
211 } else {
212 permissions = ImmutableList.of(Manifest.permission.RECORD_AUDIO);
213 }
214 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
215 putScreenInCallMode();
216 checkRecorderAndAcceptCall();
217 }
218 }
219
220 private void checkRecorderAndAcceptCall() {
221 checkMicrophoneAvailability();
222 try {
223 requireRtpConnection().acceptCall();
224 } catch (final IllegalStateException e) {
225 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
226 }
227 }
228
229 private void checkMicrophoneAvailability() {
230 new Thread(() -> {
231 final long start = SystemClock.elapsedRealtime();
232 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
233 final long stop = SystemClock.elapsedRealtime();
234 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
235 if (isMicrophoneAvailable) {
236 return;
237 }
238 runOnUiThread(() -> Toast.makeText(this, R.string.microphone_unavailable, Toast.LENGTH_LONG).show());
239 }
240 ).start();
241 }
242
243 private void putScreenInCallMode() {
244 putScreenInCallMode(requireRtpConnection().getMedia());
245 }
246
247 private void putScreenInCallMode(final Set<Media> media) {
248 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
249 if (!media.contains(Media.VIDEO)) {
250 final JingleRtpConnection rtpConnection = rtpConnectionReference != null ? rtpConnectionReference.get() : null;
251 final AppRTCAudioManager audioManager = rtpConnection == null ? null : rtpConnection.getAudioManager();
252 if (audioManager == null || audioManager.getSelectedAudioDevice() == AppRTCAudioManager.AudioDevice.EARPIECE) {
253 acquireProximityWakeLock();
254 }
255 }
256 }
257
258 @SuppressLint("WakelockTimeout")
259 private void acquireProximityWakeLock() {
260 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
261 if (powerManager == null) {
262 Log.e(Config.LOGTAG, "power manager not available");
263 return;
264 }
265 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
266 if (this.mProximityWakeLock == null) {
267 this.mProximityWakeLock = powerManager.newWakeLock(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
268 }
269 if (!this.mProximityWakeLock.isHeld()) {
270 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
271 this.mProximityWakeLock.acquire();
272 }
273 }
274 }
275
276 private void releaseProximityWakeLock() {
277 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
278 Log.d(Config.LOGTAG, "releasing proximity wake lock");
279 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
280 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
281 } else {
282 this.mProximityWakeLock.release();
283 }
284 this.mProximityWakeLock = null;
285 }
286 }
287
288 private void putProximityWakeLockInProperState(final AppRTCAudioManager.AudioDevice audioDevice) {
289 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
290 acquireProximityWakeLock();
291 } else {
292 releaseProximityWakeLock();
293 }
294 }
295
296 @Override
297 protected void refreshUiReal() {
298
299 }
300
301 @Override
302 public void onNewIntent(final Intent intent) {
303 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
304 super.onNewIntent(intent);
305 setIntent(intent);
306 if (xmppConnectionService == null) {
307 Log.d(Config.LOGTAG, "RtpSessionActivity: background service wasn't bound in onNewIntent()");
308 return;
309 }
310 final Account account = extractAccount(intent);
311 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
312 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
313 if (sessionId != null) {
314 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
315 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
316 return;
317 }
318 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
319 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
320 requestPermissionsAndAcceptCall();
321 resetIntent(intent.getExtras());
322 }
323 } else {
324 throw new IllegalStateException("received onNewIntent without sessionId");
325 }
326 }
327
328 @Override
329 void onBackendConnected() {
330 final Intent intent = getIntent();
331 final String action = intent.getAction();
332 final Account account = extractAccount(intent);
333 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
334 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
335 if (sessionId != null) {
336 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
337 return;
338 }
339 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
340 Log.d(Config.LOGTAG, "intent action was accept");
341 requestPermissionsAndAcceptCall();
342 resetIntent(intent.getExtras());
343 }
344 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
345 proposeJingleRtpSession(account, with, actionToMedia(action));
346 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
347 } else if (Intent.ACTION_VIEW.equals(action)) {
348 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
349 if (extraLastState != null) {
350 Log.d(Config.LOGTAG, "restored last state from intent extra");
351 RtpEndUserState state = RtpEndUserState.valueOf(extraLastState);
352 updateButtonConfiguration(state);
353 updateStateDisplay(state);
354 updateProfilePicture(state);
355 invalidateOptionsMenu();
356 }
357 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
358 }
359 }
360
361 private void proposeJingleRtpSession(final Account account, final Jid with, final Set<Media> media) {
362 checkMicrophoneAvailability();
363 if (with.isBareJid()) {
364 xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media);
365 } else {
366 final String sessionId = xmppConnectionService.getJingleConnectionManager().initializeRtpSession(account, with, media);
367 initializeActivityWithRunningRtpSession(account, with, sessionId);
368 resetIntent(account, with, sessionId);
369 }
370 putScreenInCallMode(media);
371 }
372
373 @Override
374 public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
375 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
376 if (PermissionUtils.allGranted(grantResults)) {
377 if (requestCode == REQUEST_ACCEPT_CALL) {
378 checkRecorderAndAcceptCall();
379 }
380 } else {
381 @StringRes int res;
382 final String firstDenied = getFirstDenied(grantResults, permissions);
383 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
384 res = R.string.no_microphone_permission;
385 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
386 res = R.string.no_camera_permission;
387 } else {
388 throw new IllegalStateException("Invalid permission result request");
389 }
390 Toast.makeText(this, res, Toast.LENGTH_SHORT).show();
391 }
392 }
393
394 @Override
395 public void onStart() {
396 super.onStart();
397 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
398 }
399
400 @Override
401 public void onStop() {
402 mHandler.removeCallbacks(mTickExecutor);
403 binding.remoteVideo.release();
404 binding.localVideo.release();
405 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
406 final JingleRtpConnection jingleRtpConnection = weakReference == null ? null : weakReference.get();
407 if (jingleRtpConnection != null) {
408 releaseVideoTracks(jingleRtpConnection);
409 }
410 releaseProximityWakeLock();
411 super.onStop();
412 }
413
414 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
415 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
416 if (remoteVideo.isPresent()) {
417 remoteVideo.get().removeSink(binding.remoteVideo);
418 }
419 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
420 if (localVideo.isPresent()) {
421 localVideo.get().removeSink(binding.localVideo);
422 }
423 }
424
425 @Override
426 public void onBackPressed() {
427 super.onBackPressed();
428 endCall();
429 }
430
431 @Override
432 public void onUserLeaveHint() {
433 super.onUserLeaveHint();
434 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
435 if (shouldBePictureInPicture()) {
436 startPictureInPicture();
437 return;
438 }
439 }
440 retractSessionProposal();
441 }
442
443 @RequiresApi(api = Build.VERSION_CODES.O)
444 private void startPictureInPicture() {
445 try {
446 enterPictureInPictureMode(
447 new PictureInPictureParams.Builder()
448 .setAspectRatio(new Rational(10, 16))
449 .build()
450 );
451 } catch (final IllegalStateException e) {
452 //this sometimes happens on Samsung phones (possibly when Knox is enabled)
453 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
454 }
455 }
456
457 private boolean deviceSupportsPictureInPicture() {
458 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
459 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
460 } else {
461 return false;
462 }
463 }
464
465 private boolean shouldBePictureInPicture() {
466 try {
467 final JingleRtpConnection rtpConnection = requireRtpConnection();
468 return rtpConnection.getMedia().contains(Media.VIDEO) && Arrays.asList(
469 RtpEndUserState.ACCEPTING_CALL,
470 RtpEndUserState.CONNECTING,
471 RtpEndUserState.CONNECTED
472 ).contains(rtpConnection.getEndUserState());
473 } catch (final IllegalStateException e) {
474 return false;
475 }
476 }
477
478 private boolean initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
479 final WeakReference<JingleRtpConnection> reference = xmppConnectionService.getJingleConnectionManager()
480 .findJingleRtpConnection(account, with, sessionId);
481 if (reference == null || reference.get() == null) {
482 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = xmppConnectionService
483 .getJingleConnectionManager().getTerminalSessionState(with, sessionId);
484 if (terminatedRtpSession == null) {
485 throw new IllegalStateException("failed to initialize activity with running rtp session. session not found");
486 }
487 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
488 return true;
489 }
490 this.rtpConnectionReference = reference;
491 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
492 if (currentState == RtpEndUserState.ENDED) {
493 reference.get().throwStateTransitionException();
494 finish();
495 return true;
496 }
497 final Set<Media> media = getMedia();
498 if (currentState == RtpEndUserState.INCOMING_CALL) {
499 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
500 }
501 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(requireRtpConnection().getState())) {
502 putScreenInCallMode();
503 }
504 binding.with.setText(getWith().getDisplayName());
505 updateVideoViews(currentState);
506 updateStateDisplay(currentState, media);
507 updateButtonConfiguration(currentState, media);
508 updateProfilePicture(currentState);
509 invalidateOptionsMenu();
510 return false;
511 }
512
513 private void initializeWithTerminatedSessionState(final Account account, final Jid with, final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
514 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
515 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
516 finish();
517 return;
518 }
519 RtpEndUserState state = terminatedRtpSession.state;
520 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
521 updateButtonConfiguration(state);
522 updateStateDisplay(state);
523 updateProfilePicture(state);
524 updateCallDuration();
525 invalidateOptionsMenu();
526 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
527 }
528
529 private void reInitializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
530 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
531 resetIntent(account, with, sessionId);
532 }
533
534 private void resetIntent(final Account account, final Jid with, final String sessionId) {
535 final Intent intent = new Intent(Intent.ACTION_VIEW);
536 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
537 intent.putExtra(EXTRA_WITH, with.toEscapedString());
538 intent.putExtra(EXTRA_SESSION_ID, sessionId);
539 setIntent(intent);
540 }
541
542 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
543 surfaceViewRenderer.setVisibility(View.VISIBLE);
544 try {
545 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
546 } catch (IllegalStateException e) {
547 Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
548 }
549 surfaceViewRenderer.setEnableHardwareScaler(true);
550 }
551
552 private void updateStateDisplay(final RtpEndUserState state) {
553 updateStateDisplay(state, Collections.emptySet());
554 }
555
556 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media) {
557 switch (state) {
558 case INCOMING_CALL:
559 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
560 if (media.contains(Media.VIDEO)) {
561 setTitle(R.string.rtp_state_incoming_video_call);
562 } else {
563 setTitle(R.string.rtp_state_incoming_call);
564 }
565 break;
566 case CONNECTING:
567 setTitle(R.string.rtp_state_connecting);
568 break;
569 case CONNECTED:
570 setTitle(R.string.rtp_state_connected);
571 break;
572 case ACCEPTING_CALL:
573 setTitle(R.string.rtp_state_accepting_call);
574 break;
575 case ENDING_CALL:
576 setTitle(R.string.rtp_state_ending_call);
577 break;
578 case FINDING_DEVICE:
579 setTitle(R.string.rtp_state_finding_device);
580 break;
581 case RINGING:
582 setTitle(R.string.rtp_state_ringing);
583 break;
584 case DECLINED_OR_BUSY:
585 setTitle(R.string.rtp_state_declined_or_busy);
586 break;
587 case CONNECTIVITY_ERROR:
588 setTitle(R.string.rtp_state_connectivity_error);
589 break;
590 case CONNECTIVITY_LOST_ERROR:
591 setTitle(R.string.rtp_state_connectivity_lost_error);
592 break;
593 case RETRACTED:
594 setTitle(R.string.rtp_state_retracted);
595 break;
596 case APPLICATION_ERROR:
597 setTitle(R.string.rtp_state_application_failure);
598 break;
599 case ENDED:
600 throw new IllegalStateException("Activity should have called finishAndReleaseWakeLock();");
601 default:
602 throw new IllegalStateException(String.format("State %s has not been handled in UI", state));
603 }
604 }
605
606 private void updateProfilePicture(final RtpEndUserState state) {
607 updateProfilePicture(state, null);
608 }
609
610 private void updateProfilePicture(final RtpEndUserState state, final Contact contact) {
611 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
612 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
613 if (show) {
614 binding.contactPhoto.setVisibility(View.VISIBLE);
615 if (contact == null) {
616 AvatarWorkerTask.loadAvatar(getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
617 } else {
618 AvatarWorkerTask.loadAvatar(contact, binding.contactPhoto, R.dimen.publish_avatar_size);
619 }
620 } else {
621 binding.contactPhoto.setVisibility(View.GONE);
622 }
623 } else {
624 binding.contactPhoto.setVisibility(View.GONE);
625 }
626 }
627
628 private Set<Media> getMedia() {
629 return requireRtpConnection().getMedia();
630 }
631
632 private void updateButtonConfiguration(final RtpEndUserState state) {
633 updateButtonConfiguration(state, Collections.emptySet());
634 }
635
636 @SuppressLint("RestrictedApi")
637 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
638 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
639 this.binding.rejectCall.setVisibility(View.INVISIBLE);
640 this.binding.endCall.setVisibility(View.INVISIBLE);
641 this.binding.acceptCall.setVisibility(View.INVISIBLE);
642 } else if (state == RtpEndUserState.INCOMING_CALL) {
643 this.binding.rejectCall.setOnClickListener(this::rejectCall);
644 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
645 this.binding.rejectCall.setVisibility(View.VISIBLE);
646 this.binding.endCall.setVisibility(View.INVISIBLE);
647 this.binding.acceptCall.setOnClickListener(this::acceptCall);
648 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
649 this.binding.acceptCall.setVisibility(View.VISIBLE);
650 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
651 this.binding.rejectCall.setOnClickListener(this::exit);
652 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
653 this.binding.rejectCall.setVisibility(View.VISIBLE);
654 this.binding.endCall.setVisibility(View.INVISIBLE);
655 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
656 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
657 this.binding.acceptCall.setVisibility(View.VISIBLE);
658 } else if (asList(
659 RtpEndUserState.CONNECTIVITY_ERROR,
660 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
661 RtpEndUserState.APPLICATION_ERROR,
662 RtpEndUserState.RETRACTED
663 ).contains(state)) {
664 this.binding.rejectCall.setOnClickListener(this::exit);
665 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
666 this.binding.rejectCall.setVisibility(View.VISIBLE);
667 this.binding.endCall.setVisibility(View.INVISIBLE);
668 this.binding.acceptCall.setOnClickListener(this::retry);
669 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
670 this.binding.acceptCall.setVisibility(View.VISIBLE);
671 } else {
672 this.binding.rejectCall.setVisibility(View.INVISIBLE);
673 this.binding.endCall.setOnClickListener(this::endCall);
674 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
675 this.binding.endCall.setVisibility(View.VISIBLE);
676 this.binding.acceptCall.setVisibility(View.INVISIBLE);
677 }
678 updateInCallButtonConfiguration(state, media);
679 }
680
681 private boolean isPictureInPicture() {
682 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
683 return isInPictureInPictureMode();
684 } else {
685 return false;
686 }
687 }
688
689 private void updateInCallButtonConfiguration() {
690 updateInCallButtonConfiguration(requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
691 }
692
693 @SuppressLint("RestrictedApi")
694 private void updateInCallButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
695 if (state == RtpEndUserState.CONNECTED && !isPictureInPicture()) {
696 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
697 if (media.contains(Media.VIDEO)) {
698 final JingleRtpConnection rtpConnection = requireRtpConnection();
699 updateInCallButtonConfigurationVideo(rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
700 } else {
701 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
702 updateInCallButtonConfigurationSpeaker(
703 audioManager.getSelectedAudioDevice(),
704 audioManager.getAudioDevices().size()
705 );
706 this.binding.inCallActionFarRight.setVisibility(View.GONE);
707 }
708 if (media.contains(Media.AUDIO)) {
709 updateInCallButtonConfigurationMicrophone(requireRtpConnection().isMicrophoneEnabled());
710 } else {
711 this.binding.inCallActionLeft.setVisibility(View.GONE);
712 }
713 } else {
714 this.binding.inCallActionLeft.setVisibility(View.GONE);
715 this.binding.inCallActionRight.setVisibility(View.GONE);
716 this.binding.inCallActionFarRight.setVisibility(View.GONE);
717 }
718 }
719
720 @SuppressLint("RestrictedApi")
721 private void updateInCallButtonConfigurationSpeaker(final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
722 switch (selectedAudioDevice) {
723 case EARPIECE:
724 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_black_24dp);
725 if (numberOfChoices >= 2) {
726 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
727 } else {
728 this.binding.inCallActionRight.setOnClickListener(null);
729 this.binding.inCallActionRight.setClickable(false);
730 }
731 break;
732 case WIRED_HEADSET:
733 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
734 this.binding.inCallActionRight.setOnClickListener(null);
735 this.binding.inCallActionRight.setClickable(false);
736 break;
737 case SPEAKER_PHONE:
738 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
739 if (numberOfChoices >= 2) {
740 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
741 } else {
742 this.binding.inCallActionRight.setOnClickListener(null);
743 this.binding.inCallActionRight.setClickable(false);
744 }
745 break;
746 case BLUETOOTH:
747 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_black_24dp);
748 this.binding.inCallActionRight.setOnClickListener(null);
749 this.binding.inCallActionRight.setClickable(false);
750 break;
751 }
752 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
753 }
754
755 @SuppressLint("RestrictedApi")
756 private void updateInCallButtonConfigurationVideo(final boolean videoEnabled, final boolean isCameraSwitchable) {
757 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
758 if (isCameraSwitchable) {
759 this.binding.inCallActionFarRight.setImageResource(R.drawable.ic_flip_camera_android_black_24dp);
760 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
761 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
762 } else {
763 this.binding.inCallActionFarRight.setVisibility(View.GONE);
764 }
765 if (videoEnabled) {
766 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
767 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
768 } else {
769 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
770 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
771 }
772 }
773
774 private void switchCamera(final View view) {
775 Futures.addCallback(requireRtpConnection().switchCamera(), new FutureCallback<Boolean>() {
776 @Override
777 public void onSuccess(@NullableDecl Boolean isFrontCamera) {
778 binding.localVideo.setMirror(isFrontCamera);
779 }
780
781 @Override
782 public void onFailure(@NonNull final Throwable throwable) {
783 Log.d(Config.LOGTAG, "could not switch camera", Throwables.getRootCause(throwable));
784 Toast.makeText(RtpSessionActivity.this, R.string.could_not_switch_camera, Toast.LENGTH_LONG).show();
785 }
786 }, MainThreadExecutor.getInstance());
787 }
788
789 private void enableVideo(View view) {
790 requireRtpConnection().setVideoEnabled(true);
791 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
792 }
793
794 private void disableVideo(View view) {
795 requireRtpConnection().setVideoEnabled(false);
796 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
797
798 }
799
800 @SuppressLint("RestrictedApi")
801 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
802 if (microphoneEnabled) {
803 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
804 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
805 } else {
806 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
807 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
808 }
809 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
810 }
811
812 private void updateCallDuration() {
813 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
814 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
815 this.binding.duration.setVisibility(View.GONE);
816 return;
817 }
818 final long rtpConnectionStarted = connection.getRtpConnectionStarted();
819 final long rtpConnectionEnded = connection.getRtpConnectionEnded();
820 if (rtpConnectionStarted != 0) {
821 final long ended = rtpConnectionEnded == 0 ? SystemClock.elapsedRealtime() : rtpConnectionEnded;
822 this.binding.duration.setText(TimeFrameUtils.formatTimePassed(rtpConnectionStarted, ended, false));
823 this.binding.duration.setVisibility(View.VISIBLE);
824 } else {
825 this.binding.duration.setVisibility(View.GONE);
826 }
827 }
828
829 private void updateVideoViews(final RtpEndUserState state) {
830 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
831 binding.localVideo.setVisibility(View.GONE);
832 binding.localVideo.release();
833 binding.remoteVideo.setVisibility(View.GONE);
834 binding.remoteVideo.release();
835 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
836 if (isPictureInPicture()) {
837 binding.appBarLayout.setVisibility(View.GONE);
838 binding.pipPlaceholder.setVisibility(View.VISIBLE);
839 if (state == RtpEndUserState.APPLICATION_ERROR || state == RtpEndUserState.CONNECTIVITY_ERROR) {
840 binding.pipWarning.setVisibility(View.VISIBLE);
841 binding.pipWaiting.setVisibility(View.GONE);
842 } else {
843 binding.pipWarning.setVisibility(View.GONE);
844 binding.pipWaiting.setVisibility(View.GONE);
845 }
846 } else {
847 binding.appBarLayout.setVisibility(View.VISIBLE);
848 binding.pipPlaceholder.setVisibility(View.GONE);
849 }
850 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
851 return;
852 }
853 if (isPictureInPicture() && (state == RtpEndUserState.CONNECTING || state == RtpEndUserState.ACCEPTING_CALL)) {
854 binding.localVideo.setVisibility(View.GONE);
855 binding.remoteVideo.setVisibility(View.GONE);
856 binding.appBarLayout.setVisibility(View.GONE);
857 binding.pipPlaceholder.setVisibility(View.VISIBLE);
858 binding.pipWarning.setVisibility(View.GONE);
859 binding.pipWaiting.setVisibility(View.VISIBLE);
860 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
861 return;
862 }
863 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
864 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
865 ensureSurfaceViewRendererIsSetup(binding.localVideo);
866 //paint local view over remote view
867 binding.localVideo.setZOrderMediaOverlay(true);
868 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
869 addSink(localVideoTrack.get(), binding.localVideo);
870 } else {
871 binding.localVideo.setVisibility(View.GONE);
872 }
873 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
874 if (remoteVideoTrack.isPresent()) {
875 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
876 addSink(remoteVideoTrack.get(), binding.remoteVideo);
877 if (state == RtpEndUserState.CONNECTED) {
878 binding.appBarLayout.setVisibility(View.GONE);
879 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
880 } else {
881 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
882 binding.remoteVideo.setVisibility(View.GONE);
883 }
884 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
885 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
886 } else {
887 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
888 }
889 } else {
890 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
891 binding.remoteVideo.setVisibility(View.GONE);
892 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
893 }
894 }
895
896 private Optional<VideoTrack> getLocalVideoTrack() {
897 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
898 if (connection == null) {
899 return Optional.absent();
900 }
901 return connection.getLocalVideoTrack();
902 }
903
904 private Optional<VideoTrack> getRemoteVideoTrack() {
905 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
906 if (connection == null) {
907 return Optional.absent();
908 }
909 return connection.getRemoteVideoTrack();
910 }
911
912 private void disableMicrophone(View view) {
913 final JingleRtpConnection rtpConnection = requireRtpConnection();
914 if (rtpConnection.setMicrophoneEnabled(false)) {
915 updateInCallButtonConfiguration();
916 }
917 }
918
919 private void enableMicrophone(View view) {
920 final JingleRtpConnection rtpConnection = requireRtpConnection();
921 if (rtpConnection.setMicrophoneEnabled(true)) {
922 updateInCallButtonConfiguration();
923 }
924 }
925
926 private void switchToEarpiece(View view) {
927 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
928 acquireProximityWakeLock();
929 }
930
931 private void switchToSpeaker(View view) {
932 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
933 releaseProximityWakeLock();
934 }
935
936 private void retry(View view) {
937 final Intent intent = getIntent();
938 final Account account = extractAccount(intent);
939 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
940 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
941 final String action = intent.getAction();
942 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
943 this.rtpConnectionReference = null;
944 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
945 proposeJingleRtpSession(account, with, media);
946 }
947
948 private void exit(final View view) {
949 finish();
950 }
951
952 private void recordVoiceMail(final View view) {
953 final Intent intent = getIntent();
954 final Account account = extractAccount(intent);
955 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
956 final Conversation conversation = xmppConnectionService.findOrCreateConversation(account, with, false, true);
957 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
958 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
959 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
960 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
961 launchIntent.putExtra(ConversationsActivity.EXTRA_POST_INIT_ACTION, ConversationsActivity.POST_ACTION_RECORD_VOICE);
962 startActivity(launchIntent);
963 finish();
964 }
965
966 private Contact getWith() {
967 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
968 final Account account = id.account;
969 return account.getRoster().getContact(id.with);
970 }
971
972 private JingleRtpConnection requireRtpConnection() {
973 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
974 if (connection == null) {
975 throw new IllegalStateException("No RTP connection found");
976 }
977 return connection;
978 }
979
980 @Override
981 public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) {
982 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
983 if (END_CARD.contains(state)) {
984 Log.d(Config.LOGTAG, "end card reached");
985 releaseProximityWakeLock();
986 runOnUiThread(() -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
987 }
988 if (with.isBareJid()) {
989 updateRtpSessionProposalState(account, with, state);
990 return;
991 }
992 if (this.rtpConnectionReference == null) {
993 if (END_CARD.contains(state)) {
994 Log.d(Config.LOGTAG, "not reinitializing session");
995 return;
996 }
997 //this happens when going from proposed session to actual session
998 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
999 return;
1000 }
1001 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1002 final Set<Media> media = getMedia();
1003 final Contact contact = getWith();
1004 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1005 if (state == RtpEndUserState.ENDED) {
1006 finish();
1007 return;
1008 }
1009 runOnUiThread(() -> {
1010 updateStateDisplay(state, media);
1011 updateButtonConfiguration(state, media);
1012 updateVideoViews(state);
1013 updateProfilePicture(state, contact);
1014 invalidateOptionsMenu();
1015 });
1016 if (END_CARD.contains(state)) {
1017 final JingleRtpConnection rtpConnection = requireRtpConnection();
1018 resetIntent(account, with, state, rtpConnection.getMedia());
1019 releaseVideoTracks(rtpConnection);
1020 this.rtpConnectionReference = null;
1021 }
1022 } else {
1023 Log.d(Config.LOGTAG, "received update for other rtp session");
1024 }
1025 }
1026
1027 @Override
1028 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1029 Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices);
1030 try {
1031 if (getMedia().contains(Media.VIDEO)) {
1032 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1033 return;
1034 }
1035 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1036 if (endUserState == RtpEndUserState.CONNECTED) {
1037 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1038 updateInCallButtonConfigurationSpeaker(
1039 audioManager.getSelectedAudioDevice(),
1040 audioManager.getAudioDevices().size()
1041 );
1042 } else if (END_CARD.contains(endUserState)) {
1043 Log.d(Config.LOGTAG, "onAudioDeviceChanged() nothing to do because end card has been reached");
1044 } else {
1045 putProximityWakeLockInProperState(selectedAudioDevice);
1046 }
1047 } catch (IllegalStateException e) {
1048 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1049 }
1050 }
1051
1052 private void updateRtpSessionProposalState(final Account account, final Jid with, final RtpEndUserState state) {
1053 final Intent currentIntent = getIntent();
1054 final String withExtra = currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1055 if (withExtra == null) {
1056 return;
1057 }
1058 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1059 runOnUiThread(() -> {
1060 updateStateDisplay(state);
1061 updateButtonConfiguration(state);
1062 updateProfilePicture(state);
1063 invalidateOptionsMenu();
1064 });
1065 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1066 }
1067 }
1068
1069 private void resetIntent(final Bundle extras) {
1070 final Intent intent = new Intent(Intent.ACTION_VIEW);
1071 intent.putExtras(extras);
1072 setIntent(intent);
1073 }
1074
1075 private void resetIntent(final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1076 final Intent intent = new Intent(Intent.ACTION_VIEW);
1077 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1078 if (account.getRoster().getContact(with).getPresences().anySupport(Namespace.JINGLE_MESSAGE)) {
1079 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1080 } else {
1081 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1082 }
1083 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1084 intent.putExtra(EXTRA_LAST_ACTION, media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1085 setIntent(intent);
1086 }
1087}