1package eu.siacs.conversations.ui;
2
3import android.Manifest;
4import android.annotation.SuppressLint;
5import android.app.PictureInPictureParams;
6import android.content.Context;
7import android.content.Intent;
8import android.content.pm.PackageManager;
9import android.databinding.DataBindingUtil;
10import android.os.Build;
11import android.os.Bundle;
12import android.os.Handler;
13import android.os.PowerManager;
14import android.os.SystemClock;
15import android.support.annotation.NonNull;
16import android.support.annotation.RequiresApi;
17import android.support.annotation.StringRes;
18import android.util.Log;
19import android.util.Rational;
20import android.view.View;
21import android.view.WindowManager;
22import android.widget.Toast;
23
24import com.google.common.base.Optional;
25import com.google.common.base.Preconditions;
26import com.google.common.base.Throwables;
27import com.google.common.collect.ImmutableList;
28import com.google.common.collect.ImmutableSet;
29import com.google.common.util.concurrent.FutureCallback;
30import com.google.common.util.concurrent.Futures;
31
32import org.checkerframework.checker.nullness.compatqual.NullableDecl;
33import org.webrtc.SurfaceViewRenderer;
34import org.webrtc.VideoTrack;
35
36import java.lang.ref.WeakReference;
37import java.util.Arrays;
38import java.util.Collections;
39import java.util.List;
40import java.util.Set;
41
42import eu.siacs.conversations.Config;
43import eu.siacs.conversations.R;
44import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
45import eu.siacs.conversations.entities.Account;
46import eu.siacs.conversations.entities.Contact;
47import eu.siacs.conversations.entities.Conversation;
48import eu.siacs.conversations.services.AppRTCAudioManager;
49import eu.siacs.conversations.services.XmppConnectionService;
50import eu.siacs.conversations.ui.util.AvatarWorkerTask;
51import eu.siacs.conversations.ui.util.MainThreadExecutor;
52import eu.siacs.conversations.utils.PermissionUtils;
53import eu.siacs.conversations.utils.TimeFrameUtils;
54import eu.siacs.conversations.xml.Namespace;
55import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
56import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
57import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
58import eu.siacs.conversations.xmpp.jingle.Media;
59import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
60import eu.siacs.conversations.xmpp.Jid;
61
62import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
63import static java.util.Arrays.asList;
64
65public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate {
66
67 public static final String EXTRA_WITH = "with";
68 public static final String EXTRA_SESSION_ID = "session_id";
69 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
70 public static final String EXTRA_LAST_ACTION = "last_action";
71 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
72 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
73 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
74
75 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
76
77 private static final List<RtpEndUserState> END_CARD = Arrays.asList(
78 RtpEndUserState.APPLICATION_ERROR,
79 RtpEndUserState.DECLINED_OR_BUSY,
80 RtpEndUserState.CONNECTIVITY_ERROR,
81 RtpEndUserState.RETRACTED
82 );
83 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
84 private static final int REQUEST_ACCEPT_CALL = 0x1111;
85 private WeakReference<JingleRtpConnection> rtpConnectionReference;
86
87 private ActivityRtpSessionBinding binding;
88 private PowerManager.WakeLock mProximityWakeLock;
89
90 private Handler mHandler = new Handler();
91 private Runnable mTickExecutor = new Runnable() {
92 @Override
93 public void run() {
94 updateCallDuration();
95 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
96 }
97 };
98
99 private static Set<Media> actionToMedia(final String action) {
100 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
101 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
102 } else {
103 return ImmutableSet.of(Media.AUDIO);
104 }
105 }
106
107 private static void addSink(final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
108 try {
109 videoTrack.addSink(surfaceViewRenderer);
110 } catch (final IllegalStateException e) {
111 Log.e(Config.LOGTAG, "possible race condition on trying to display video track. ignoring", e);
112 }
113 }
114
115 @Override
116 public void onCreate(Bundle savedInstanceState) {
117 super.onCreate(savedInstanceState);
118 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
119 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
120 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
121 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
122 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
123 setSupportActionBar(binding.toolbar);
124 }
125
126 private void endCall(View view) {
127 endCall();
128 }
129
130 private void endCall() {
131 if (this.rtpConnectionReference == null) {
132 retractSessionProposal();
133 finish();
134 } else {
135 requireRtpConnection().endCall();
136 }
137 }
138
139 private void retractSessionProposal() {
140 final Intent intent = getIntent();
141 final String action = intent.getAction();
142 final Account account = extractAccount(intent);
143 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
144 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
145 if (!Intent.ACTION_VIEW.equals(action) || state == null || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
146 resetIntent(account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
147 }
148 xmppConnectionService.getJingleConnectionManager().retractSessionProposal(account, with.asBareJid());
149 }
150
151 private void rejectCall(View view) {
152 requireRtpConnection().rejectCall();
153 finish();
154 }
155
156 private void acceptCall(View view) {
157 requestPermissionsAndAcceptCall();
158 }
159
160 private void requestPermissionsAndAcceptCall() {
161 final List<String> permissions;
162 if (getMedia().contains(Media.VIDEO)) {
163 permissions = ImmutableList.of(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO);
164 } else {
165 permissions = ImmutableList.of(Manifest.permission.RECORD_AUDIO);
166 }
167 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
168 putScreenInCallMode();
169 checkRecorderAndAcceptCall();
170 }
171 }
172
173 private void checkRecorderAndAcceptCall() {
174 checkMicrophoneAvailability();
175 try {
176 requireRtpConnection().acceptCall();
177 } catch (final IllegalStateException e) {
178 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
179 }
180 }
181
182 private void checkMicrophoneAvailability() {
183 new Thread(() -> {
184 final long start = SystemClock.elapsedRealtime();
185 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
186 final long stop = SystemClock.elapsedRealtime();
187 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
188 if (isMicrophoneAvailable) {
189 return;
190 }
191 runOnUiThread(() -> Toast.makeText(this, R.string.microphone_unavailable, Toast.LENGTH_LONG).show());
192 }
193 ).start();
194 }
195
196 private void putScreenInCallMode() {
197 putScreenInCallMode(requireRtpConnection().getMedia());
198 }
199
200 private void putScreenInCallMode(final Set<Media> media) {
201 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
202 if (!media.contains(Media.VIDEO)) {
203 final JingleRtpConnection rtpConnection = rtpConnectionReference != null ? rtpConnectionReference.get() : null;
204 final AppRTCAudioManager audioManager = rtpConnection == null ? null : rtpConnection.getAudioManager();
205 if (audioManager == null || audioManager.getSelectedAudioDevice() == AppRTCAudioManager.AudioDevice.EARPIECE) {
206 acquireProximityWakeLock();
207 }
208 }
209 }
210
211 @SuppressLint("WakelockTimeout")
212 private void acquireProximityWakeLock() {
213 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
214 if (powerManager == null) {
215 Log.e(Config.LOGTAG, "power manager not available");
216 return;
217 }
218 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
219 if (this.mProximityWakeLock == null) {
220 this.mProximityWakeLock = powerManager.newWakeLock(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
221 }
222 if (!this.mProximityWakeLock.isHeld()) {
223 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
224 this.mProximityWakeLock.acquire();
225 }
226 }
227 }
228
229 private void releaseProximityWakeLock() {
230 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
231 Log.d(Config.LOGTAG, "releasing proximity wake lock");
232 this.mProximityWakeLock.release();
233 this.mProximityWakeLock = null;
234 }
235 }
236
237 private void putProximityWakeLockInProperState() {
238 if (requireRtpConnection().getAudioManager().getSelectedAudioDevice() == AppRTCAudioManager.AudioDevice.EARPIECE) {
239 acquireProximityWakeLock();
240 } else {
241 releaseProximityWakeLock();
242 }
243 }
244
245 @Override
246 protected void refreshUiReal() {
247
248 }
249
250 @Override
251 public void onNewIntent(final Intent intent) {
252 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
253 super.onNewIntent(intent);
254 setIntent(intent);
255 if (xmppConnectionService == null) {
256 Log.d(Config.LOGTAG, "RtpSessionActivity: background service wasn't bound in onNewIntent()");
257 return;
258 }
259 final Account account = extractAccount(intent);
260 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
261 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
262 if (sessionId != null) {
263 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
264 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
265 return;
266 }
267 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
268 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
269 requestPermissionsAndAcceptCall();
270 resetIntent(intent.getExtras());
271 }
272 } else {
273 throw new IllegalStateException("received onNewIntent without sessionId");
274 }
275 }
276
277 @Override
278 void onBackendConnected() {
279 final Intent intent = getIntent();
280 final String action = intent.getAction();
281 final Account account = extractAccount(intent);
282 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
283 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
284 if (sessionId != null) {
285 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
286 return;
287 }
288 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
289 Log.d(Config.LOGTAG, "intent action was accept");
290 requestPermissionsAndAcceptCall();
291 resetIntent(intent.getExtras());
292 }
293 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
294 proposeJingleRtpSession(account, with, actionToMedia(action));
295 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
296 } else if (Intent.ACTION_VIEW.equals(action)) {
297 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
298 if (extraLastState != null) {
299 Log.d(Config.LOGTAG, "restored last state from intent extra");
300 RtpEndUserState state = RtpEndUserState.valueOf(extraLastState);
301 updateButtonConfiguration(state);
302 updateStateDisplay(state);
303 updateProfilePicture(state);
304 }
305 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
306 }
307 }
308
309 private void proposeJingleRtpSession(final Account account, final Jid with, final Set<Media> media) {
310 checkMicrophoneAvailability();
311 if (with.isBareJid()) {
312 xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media);
313 } else {
314 final String sessionId = xmppConnectionService.getJingleConnectionManager().initializeRtpSession(account, with, media);
315 initializeActivityWithRunningRtpSession(account, with, sessionId);
316 resetIntent(account, with, sessionId);
317 }
318 putScreenInCallMode(media);
319 }
320
321 @Override
322 public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
323 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
324 if (PermissionUtils.allGranted(grantResults)) {
325 if (requestCode == REQUEST_ACCEPT_CALL) {
326 checkRecorderAndAcceptCall();
327 }
328 } else {
329 @StringRes int res;
330 final String firstDenied = getFirstDenied(grantResults, permissions);
331 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
332 res = R.string.no_microphone_permission;
333 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
334 res = R.string.no_camera_permission;
335 } else {
336 throw new IllegalStateException("Invalid permission result request");
337 }
338 Toast.makeText(this, res, Toast.LENGTH_SHORT).show();
339 }
340 }
341
342 @Override
343 public void onStart() {
344 super.onStart();
345 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
346 }
347
348 @Override
349 public void onStop() {
350 mHandler.removeCallbacks(mTickExecutor);
351 binding.remoteVideo.release();
352 binding.localVideo.release();
353 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
354 final JingleRtpConnection jingleRtpConnection = weakReference == null ? null : weakReference.get();
355 if (jingleRtpConnection != null) {
356 releaseVideoTracks(jingleRtpConnection);
357 } else if (!isChangingConfigurations()) {
358 if (xmppConnectionService != null) {
359 retractSessionProposal();
360 }
361 }
362 releaseProximityWakeLock();
363 super.onStop();
364 }
365
366 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
367 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
368 if (remoteVideo.isPresent()) {
369 remoteVideo.get().removeSink(binding.remoteVideo);
370 }
371 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
372 if (localVideo.isPresent()) {
373 localVideo.get().removeSink(binding.localVideo);
374 }
375 }
376
377 @Override
378 public void onBackPressed() {
379 endCall();
380 super.onBackPressed();
381 }
382
383 @Override
384 public void onUserLeaveHint() {
385 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
386 if (shouldBePictureInPicture()) {
387 startPictureInPicture();
388 }
389 }
390 }
391
392 @RequiresApi(api = Build.VERSION_CODES.O)
393 private void startPictureInPicture() {
394 try {
395 enterPictureInPictureMode(
396 new PictureInPictureParams.Builder()
397 .setAspectRatio(new Rational(10, 16))
398 .build()
399 );
400 } catch (IllegalStateException e) {
401 //this sometimes happens on Samsung phones (possibly when Knox is enabled)
402 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
403 }
404 }
405
406 private boolean deviceSupportsPictureInPicture() {
407 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
408 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
409 } else {
410 return false;
411 }
412 }
413
414 private boolean shouldBePictureInPicture() {
415 try {
416 final JingleRtpConnection rtpConnection = requireRtpConnection();
417 return rtpConnection.getMedia().contains(Media.VIDEO) && Arrays.asList(
418 RtpEndUserState.ACCEPTING_CALL,
419 RtpEndUserState.CONNECTING,
420 RtpEndUserState.CONNECTED
421 ).contains(rtpConnection.getEndUserState());
422 } catch (IllegalStateException e) {
423 return false;
424 }
425 }
426
427 private boolean initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
428 final WeakReference<JingleRtpConnection> reference = xmppConnectionService.getJingleConnectionManager()
429 .findJingleRtpConnection(account, with, sessionId);
430 if (reference == null || reference.get() == null) {
431 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = xmppConnectionService
432 .getJingleConnectionManager().getTerminalSessionState(with, sessionId);
433 if (terminatedRtpSession == null) {
434 throw new IllegalStateException("failed to initialize activity with running rtp session. session not found");
435 }
436 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
437 return true;
438 }
439 this.rtpConnectionReference = reference;
440 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
441 if (currentState == RtpEndUserState.ENDED) {
442 reference.get().throwStateTransitionException();
443 finish();
444 return true;
445 }
446 final Set<Media> media = getMedia();
447 if (currentState == RtpEndUserState.INCOMING_CALL) {
448 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
449 }
450 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(requireRtpConnection().getState())) {
451 putScreenInCallMode();
452 }
453 binding.with.setText(getWith().getDisplayName());
454 updateVideoViews(currentState);
455 updateStateDisplay(currentState, media);
456 updateButtonConfiguration(currentState, media);
457 updateProfilePicture(currentState);
458 return false;
459 }
460
461 private void initializeWithTerminatedSessionState(final Account account, final Jid with, final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
462 Log.d(Config.LOGTAG,"initializeWithTerminatedSessionState()");
463 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
464 finish();
465 return;
466 }
467 RtpEndUserState state = terminatedRtpSession.state;
468 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
469 updateButtonConfiguration(state);
470 updateStateDisplay(state);
471 updateProfilePicture(state);
472 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
473 }
474
475 private void reInitializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
476 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
477 resetIntent(account, with, sessionId);
478 }
479
480 private void resetIntent(final Account account, final Jid with, final String sessionId) {
481 final Intent intent = new Intent(Intent.ACTION_VIEW);
482 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
483 intent.putExtra(EXTRA_WITH, with.toEscapedString());
484 intent.putExtra(EXTRA_SESSION_ID, sessionId);
485 setIntent(intent);
486 }
487
488 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
489 surfaceViewRenderer.setVisibility(View.VISIBLE);
490 try {
491 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
492 } catch (IllegalStateException e) {
493 Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
494 }
495 surfaceViewRenderer.setEnableHardwareScaler(true);
496 }
497
498 private void updateStateDisplay(final RtpEndUserState state) {
499 updateStateDisplay(state, Collections.emptySet());
500 }
501
502 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media) {
503 switch (state) {
504 case INCOMING_CALL:
505 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
506 if (media.contains(Media.VIDEO)) {
507 setTitle(R.string.rtp_state_incoming_video_call);
508 } else {
509 setTitle(R.string.rtp_state_incoming_call);
510 }
511 break;
512 case CONNECTING:
513 setTitle(R.string.rtp_state_connecting);
514 break;
515 case CONNECTED:
516 setTitle(R.string.rtp_state_connected);
517 break;
518 case ACCEPTING_CALL:
519 setTitle(R.string.rtp_state_accepting_call);
520 break;
521 case ENDING_CALL:
522 setTitle(R.string.rtp_state_ending_call);
523 break;
524 case FINDING_DEVICE:
525 setTitle(R.string.rtp_state_finding_device);
526 break;
527 case RINGING:
528 setTitle(R.string.rtp_state_ringing);
529 break;
530 case DECLINED_OR_BUSY:
531 setTitle(R.string.rtp_state_declined_or_busy);
532 break;
533 case CONNECTIVITY_ERROR:
534 setTitle(R.string.rtp_state_connectivity_error);
535 break;
536 case RETRACTED:
537 setTitle(R.string.rtp_state_retracted);
538 break;
539 case APPLICATION_ERROR:
540 setTitle(R.string.rtp_state_application_failure);
541 break;
542 case ENDED:
543 throw new IllegalStateException("Activity should have called finishAndReleaseWakeLock();");
544 default:
545 throw new IllegalStateException(String.format("State %s has not been handled in UI", state));
546 }
547 }
548
549 private void updateProfilePicture(final RtpEndUserState state) {
550 updateProfilePicture(state, null);
551 }
552
553 private void updateProfilePicture(final RtpEndUserState state, final Contact contact) {
554 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
555 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
556 if (show) {
557 binding.contactPhoto.setVisibility(View.VISIBLE);
558 if (contact == null) {
559 AvatarWorkerTask.loadAvatar(getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
560 } else {
561 AvatarWorkerTask.loadAvatar(contact, binding.contactPhoto, R.dimen.publish_avatar_size);
562 }
563 } else {
564 binding.contactPhoto.setVisibility(View.GONE);
565 }
566 } else {
567 binding.contactPhoto.setVisibility(View.GONE);
568 }
569 }
570
571 private Set<Media> getMedia() {
572 return requireRtpConnection().getMedia();
573 }
574
575 private void updateButtonConfiguration(final RtpEndUserState state) {
576 updateButtonConfiguration(state, Collections.emptySet());
577 }
578
579 @SuppressLint("RestrictedApi")
580 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
581 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
582 this.binding.rejectCall.setVisibility(View.INVISIBLE);
583 this.binding.endCall.setVisibility(View.INVISIBLE);
584 this.binding.acceptCall.setVisibility(View.INVISIBLE);
585 } else if (state == RtpEndUserState.INCOMING_CALL) {
586 this.binding.rejectCall.setOnClickListener(this::rejectCall);
587 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
588 this.binding.rejectCall.setVisibility(View.VISIBLE);
589 this.binding.endCall.setVisibility(View.INVISIBLE);
590 this.binding.acceptCall.setOnClickListener(this::acceptCall);
591 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
592 this.binding.acceptCall.setVisibility(View.VISIBLE);
593 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
594 this.binding.rejectCall.setOnClickListener(this::exit);
595 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
596 this.binding.rejectCall.setVisibility(View.VISIBLE);
597 this.binding.endCall.setVisibility(View.INVISIBLE);
598 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
599 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
600 this.binding.acceptCall.setVisibility(View.VISIBLE);
601 } else if (asList(RtpEndUserState.CONNECTIVITY_ERROR, RtpEndUserState.APPLICATION_ERROR, RtpEndUserState.RETRACTED).contains(state)) {
602 this.binding.rejectCall.setOnClickListener(this::exit);
603 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
604 this.binding.rejectCall.setVisibility(View.VISIBLE);
605 this.binding.endCall.setVisibility(View.INVISIBLE);
606 this.binding.acceptCall.setOnClickListener(this::retry);
607 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
608 this.binding.acceptCall.setVisibility(View.VISIBLE);
609 } else {
610 this.binding.rejectCall.setVisibility(View.INVISIBLE);
611 this.binding.endCall.setOnClickListener(this::endCall);
612 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
613 this.binding.endCall.setVisibility(View.VISIBLE);
614 this.binding.acceptCall.setVisibility(View.INVISIBLE);
615 }
616 updateInCallButtonConfiguration(state, media);
617 }
618
619 private boolean isPictureInPicture() {
620 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
621 return isInPictureInPictureMode();
622 } else {
623 return false;
624 }
625 }
626
627 private void updateInCallButtonConfiguration() {
628 updateInCallButtonConfiguration(requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
629 }
630
631 @SuppressLint("RestrictedApi")
632 private void updateInCallButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
633 if (state == RtpEndUserState.CONNECTED && !isPictureInPicture()) {
634 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
635 if (media.contains(Media.VIDEO)) {
636 final JingleRtpConnection rtpConnection = requireRtpConnection();
637 updateInCallButtonConfigurationVideo(rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
638 } else {
639 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
640 updateInCallButtonConfigurationSpeaker(
641 audioManager.getSelectedAudioDevice(),
642 audioManager.getAudioDevices().size()
643 );
644 this.binding.inCallActionFarRight.setVisibility(View.GONE);
645 }
646 if (media.contains(Media.AUDIO)) {
647 updateInCallButtonConfigurationMicrophone(requireRtpConnection().isMicrophoneEnabled());
648 } else {
649 this.binding.inCallActionLeft.setVisibility(View.GONE);
650 }
651 } else {
652 this.binding.inCallActionLeft.setVisibility(View.GONE);
653 this.binding.inCallActionRight.setVisibility(View.GONE);
654 this.binding.inCallActionFarRight.setVisibility(View.GONE);
655 }
656 }
657
658 @SuppressLint("RestrictedApi")
659 private void updateInCallButtonConfigurationSpeaker(final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
660 switch (selectedAudioDevice) {
661 case EARPIECE:
662 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_black_24dp);
663 if (numberOfChoices >= 2) {
664 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
665 } else {
666 this.binding.inCallActionRight.setOnClickListener(null);
667 this.binding.inCallActionRight.setClickable(false);
668 }
669 break;
670 case WIRED_HEADSET:
671 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
672 this.binding.inCallActionRight.setOnClickListener(null);
673 this.binding.inCallActionRight.setClickable(false);
674 break;
675 case SPEAKER_PHONE:
676 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
677 if (numberOfChoices >= 2) {
678 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
679 } else {
680 this.binding.inCallActionRight.setOnClickListener(null);
681 this.binding.inCallActionRight.setClickable(false);
682 }
683 break;
684 case BLUETOOTH:
685 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_black_24dp);
686 this.binding.inCallActionRight.setOnClickListener(null);
687 this.binding.inCallActionRight.setClickable(false);
688 break;
689 }
690 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
691 }
692
693 @SuppressLint("RestrictedApi")
694 private void updateInCallButtonConfigurationVideo(final boolean videoEnabled, final boolean isCameraSwitchable) {
695 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
696 if (isCameraSwitchable) {
697 this.binding.inCallActionFarRight.setImageResource(R.drawable.ic_flip_camera_android_black_24dp);
698 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
699 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
700 } else {
701 this.binding.inCallActionFarRight.setVisibility(View.GONE);
702 }
703 if (videoEnabled) {
704 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
705 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
706 } else {
707 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
708 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
709 }
710 }
711
712 private void switchCamera(final View view) {
713 Futures.addCallback(requireRtpConnection().switchCamera(), new FutureCallback<Boolean>() {
714 @Override
715 public void onSuccess(@NullableDecl Boolean isFrontCamera) {
716 binding.localVideo.setMirror(isFrontCamera);
717 }
718
719 @Override
720 public void onFailure(@NonNull final Throwable throwable) {
721 Log.d(Config.LOGTAG, "could not switch camera", Throwables.getRootCause(throwable));
722 Toast.makeText(RtpSessionActivity.this, R.string.could_not_switch_camera, Toast.LENGTH_LONG).show();
723 }
724 }, MainThreadExecutor.getInstance());
725 }
726
727 private void enableVideo(View view) {
728 requireRtpConnection().setVideoEnabled(true);
729 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
730 }
731
732 private void disableVideo(View view) {
733 requireRtpConnection().setVideoEnabled(false);
734 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
735
736 }
737
738 @SuppressLint("RestrictedApi")
739 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
740 if (microphoneEnabled) {
741 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
742 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
743 } else {
744 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
745 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
746 }
747 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
748 }
749
750 private void updateCallDuration() {
751 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
752 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
753 this.binding.duration.setVisibility(View.GONE);
754 return;
755 }
756 final long rtpConnectionStarted = connection.getRtpConnectionStarted();
757 final long rtpConnectionEnded = connection.getRtpConnectionEnded();
758 if (rtpConnectionStarted != 0) {
759 final long ended = rtpConnectionEnded == 0 ? SystemClock.elapsedRealtime() : rtpConnectionEnded;
760 this.binding.duration.setText(TimeFrameUtils.formatTimePassed(rtpConnectionStarted, ended, false));
761 this.binding.duration.setVisibility(View.VISIBLE);
762 } else {
763 this.binding.duration.setVisibility(View.GONE);
764 }
765 }
766
767 private void updateVideoViews(final RtpEndUserState state) {
768 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
769 binding.localVideo.setVisibility(View.GONE);
770 binding.localVideo.release();
771 binding.remoteVideo.setVisibility(View.GONE);
772 binding.remoteVideo.release();
773 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
774 if (isPictureInPicture()) {
775 binding.appBarLayout.setVisibility(View.GONE);
776 binding.pipPlaceholder.setVisibility(View.VISIBLE);
777 if (state == RtpEndUserState.APPLICATION_ERROR || state == RtpEndUserState.CONNECTIVITY_ERROR) {
778 binding.pipWarning.setVisibility(View.VISIBLE);
779 binding.pipWaiting.setVisibility(View.GONE);
780 } else {
781 binding.pipWarning.setVisibility(View.GONE);
782 binding.pipWaiting.setVisibility(View.GONE);
783 }
784 } else {
785 binding.appBarLayout.setVisibility(View.VISIBLE);
786 binding.pipPlaceholder.setVisibility(View.GONE);
787 }
788 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
789 return;
790 }
791 if (isPictureInPicture() && (state == RtpEndUserState.CONNECTING || state == RtpEndUserState.ACCEPTING_CALL)) {
792 binding.localVideo.setVisibility(View.GONE);
793 binding.remoteVideo.setVisibility(View.GONE);
794 binding.appBarLayout.setVisibility(View.GONE);
795 binding.pipPlaceholder.setVisibility(View.VISIBLE);
796 binding.pipWarning.setVisibility(View.GONE);
797 binding.pipWaiting.setVisibility(View.VISIBLE);
798 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
799 return;
800 }
801 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
802 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
803 ensureSurfaceViewRendererIsSetup(binding.localVideo);
804 //paint local view over remote view
805 binding.localVideo.setZOrderMediaOverlay(true);
806 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
807 addSink(localVideoTrack.get(), binding.localVideo);
808 } else {
809 binding.localVideo.setVisibility(View.GONE);
810 }
811 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
812 if (remoteVideoTrack.isPresent()) {
813 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
814 addSink(remoteVideoTrack.get(), binding.remoteVideo);
815 if (state == RtpEndUserState.CONNECTED) {
816 binding.appBarLayout.setVisibility(View.GONE);
817 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
818 } else {
819 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
820 binding.remoteVideo.setVisibility(View.GONE);
821 }
822 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
823 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
824 } else {
825 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
826 }
827 } else {
828 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
829 binding.remoteVideo.setVisibility(View.GONE);
830 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
831 }
832 }
833
834 private Optional<VideoTrack> getLocalVideoTrack() {
835 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
836 if (connection == null) {
837 return Optional.absent();
838 }
839 return connection.getLocalVideoTrack();
840 }
841
842 private Optional<VideoTrack> getRemoteVideoTrack() {
843 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
844 if (connection == null) {
845 return Optional.absent();
846 }
847 return connection.getRemoteVideoTrack();
848 }
849
850 private void disableMicrophone(View view) {
851 JingleRtpConnection rtpConnection = requireRtpConnection();
852 rtpConnection.setMicrophoneEnabled(false);
853 updateInCallButtonConfiguration();
854 }
855
856 private void enableMicrophone(View view) {
857 JingleRtpConnection rtpConnection = requireRtpConnection();
858 rtpConnection.setMicrophoneEnabled(true);
859 updateInCallButtonConfiguration();
860 }
861
862 private void switchToEarpiece(View view) {
863 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
864 acquireProximityWakeLock();
865 }
866
867 private void switchToSpeaker(View view) {
868 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
869 releaseProximityWakeLock();
870 }
871
872 private void retry(View view) {
873 final Intent intent = getIntent();
874 final Account account = extractAccount(intent);
875 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
876 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
877 final String action = intent.getAction();
878 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
879 this.rtpConnectionReference = null;
880 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
881 proposeJingleRtpSession(account, with, media);
882 }
883
884 private void exit(final View view) {
885 finish();
886 }
887
888 private void recordVoiceMail(final View view) {
889 final Intent intent = getIntent();
890 final Account account = extractAccount(intent);
891 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
892 final Conversation conversation = xmppConnectionService.findOrCreateConversation(account, with, false, true);
893 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
894 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
895 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
896 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
897 launchIntent.putExtra(ConversationsActivity.EXTRA_POST_INIT_ACTION, ConversationsActivity.POST_ACTION_RECORD_VOICE);
898 startActivity(launchIntent);
899 finish();
900 }
901
902 private Contact getWith() {
903 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
904 final Account account = id.account;
905 return account.getRoster().getContact(id.with);
906 }
907
908 private JingleRtpConnection requireRtpConnection() {
909 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
910 if (connection == null) {
911 throw new IllegalStateException("No RTP connection found");
912 }
913 return connection;
914 }
915
916 @Override
917 public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) {
918 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
919 if (END_CARD.contains(state)) {
920 Log.d(Config.LOGTAG, "end card reached");
921 releaseProximityWakeLock();
922 runOnUiThread(() -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
923 }
924 if (with.isBareJid()) {
925 updateRtpSessionProposalState(account, with, state);
926 return;
927 }
928 if (this.rtpConnectionReference == null) {
929 if (END_CARD.contains(state)) {
930 Log.d(Config.LOGTAG, "not reinitializing session");
931 return;
932 }
933 //this happens when going from proposed session to actual session
934 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
935 return;
936 }
937 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
938 final Set<Media> media = getMedia();
939 final Contact contact = getWith();
940 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
941 if (state == RtpEndUserState.ENDED) {
942 finish();
943 return;
944 }
945 runOnUiThread(() -> {
946 updateStateDisplay(state, media);
947 updateButtonConfiguration(state, media);
948 updateVideoViews(state);
949 updateProfilePicture(state, contact);
950 });
951 if (END_CARD.contains(state)) {
952 final JingleRtpConnection rtpConnection = requireRtpConnection();
953 resetIntent(account, with, state, rtpConnection.getMedia());
954 releaseVideoTracks(rtpConnection);
955 this.rtpConnectionReference = null;
956 }
957 } else {
958 Log.d(Config.LOGTAG, "received update for other rtp session");
959 }
960 }
961
962 @Override
963 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
964 Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices);
965 try {
966 if (getMedia().contains(Media.VIDEO)) {
967 Log.d(Config.LOGTAG, "nothing to do; in video mode");
968 return;
969 }
970 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
971 if (endUserState == RtpEndUserState.CONNECTED) {
972 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
973 updateInCallButtonConfigurationSpeaker(
974 audioManager.getSelectedAudioDevice(),
975 audioManager.getAudioDevices().size()
976 );
977 } else if (END_CARD.contains(endUserState)) {
978 Log.d(Config.LOGTAG, "onAudioDeviceChanged() nothing to do because end card has been reached");
979 } else {
980 putProximityWakeLockInProperState();
981 }
982 } catch (IllegalStateException e) {
983 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
984 }
985 }
986
987 private void updateRtpSessionProposalState(final Account account, final Jid with, final RtpEndUserState state) {
988 final Intent currentIntent = getIntent();
989 final String withExtra = currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
990 if (withExtra == null) {
991 return;
992 }
993 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
994 runOnUiThread(() -> {
995 updateStateDisplay(state);
996 updateButtonConfiguration(state);
997 updateProfilePicture(state);
998 });
999 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1000 }
1001 }
1002
1003 private void resetIntent(final Bundle extras) {
1004 final Intent intent = new Intent(Intent.ACTION_VIEW);
1005 intent.putExtras(extras);
1006 setIntent(intent);
1007 }
1008
1009 private void resetIntent(final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1010 final Intent intent = new Intent(Intent.ACTION_VIEW);
1011 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1012 if (account.getRoster().getContact(with).getPresences().anySupport(Namespace.JINGLE_MESSAGE)) {
1013 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1014 } else {
1015 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1016 }
1017 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1018 intent.putExtra(EXTRA_LAST_ACTION, media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1019 setIntent(intent);
1020 }
1021}