1package eu.siacs.conversations.ui;
2
3import static java.util.Arrays.asList;
4import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied;
5
6import android.Manifest;
7import android.annotation.SuppressLint;
8import android.app.PictureInPictureParams;
9import android.content.ActivityNotFoundException;
10import android.content.Context;
11import android.content.Intent;
12import android.content.pm.PackageManager;
13import android.os.Build;
14import android.os.Bundle;
15import android.os.Handler;
16import android.os.PowerManager;
17import android.os.SystemClock;
18import android.util.Log;
19import android.util.Rational;
20import android.view.KeyEvent;
21import android.view.Menu;
22import android.view.MenuItem;
23import android.view.View;
24import android.view.WindowManager;
25import android.widget.Toast;
26
27import androidx.annotation.NonNull;
28import androidx.annotation.RequiresApi;
29import androidx.annotation.StringRes;
30import androidx.databinding.DataBindingUtil;
31
32import com.google.common.base.Optional;
33import com.google.common.base.Preconditions;
34import com.google.common.base.Throwables;
35import com.google.common.collect.ImmutableList;
36import com.google.common.collect.ImmutableSet;
37import com.google.common.util.concurrent.FutureCallback;
38import com.google.common.util.concurrent.Futures;
39
40import org.checkerframework.checker.nullness.compatqual.NullableDecl;
41import org.jetbrains.annotations.NotNull;
42import org.webrtc.RendererCommon;
43import org.webrtc.SurfaceViewRenderer;
44import org.webrtc.VideoTrack;
45
46import java.lang.ref.WeakReference;
47import java.util.Arrays;
48import java.util.Collections;
49import java.util.List;
50import java.util.Set;
51
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.R;
54import eu.siacs.conversations.databinding.ActivityRtpSessionBinding;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.services.AppRTCAudioManager;
59import eu.siacs.conversations.services.XmppConnectionService;
60import eu.siacs.conversations.ui.util.AvatarWorkerTask;
61import eu.siacs.conversations.ui.util.MainThreadExecutor;
62import eu.siacs.conversations.ui.util.Rationals;
63import eu.siacs.conversations.utils.PermissionUtils;
64import eu.siacs.conversations.utils.TimeFrameUtils;
65import eu.siacs.conversations.xml.Namespace;
66import eu.siacs.conversations.xmpp.Jid;
67import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection;
68import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager;
69import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection;
70import eu.siacs.conversations.xmpp.jingle.Media;
71import eu.siacs.conversations.xmpp.jingle.RtpEndUserState;
72
73public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate, eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged {
74
75 public static final String EXTRA_WITH = "with";
76 public static final String EXTRA_SESSION_ID = "session_id";
77 public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
78 public static final String EXTRA_LAST_ACTION = "last_action";
79 public static final String ACTION_ACCEPT_CALL = "action_accept_call";
80 public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
81 public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
82
83 private static final int CALL_DURATION_UPDATE_INTERVAL = 333;
84
85 private static final List<RtpEndUserState> END_CARD = Arrays.asList(
86 RtpEndUserState.APPLICATION_ERROR,
87 RtpEndUserState.SECURITY_ERROR,
88 RtpEndUserState.DECLINED_OR_BUSY,
89 RtpEndUserState.CONNECTIVITY_ERROR,
90 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
91 RtpEndUserState.RETRACTED
92 );
93 private static final List<RtpEndUserState> STATES_SHOWING_HELP_BUTTON = Arrays.asList(
94 RtpEndUserState.APPLICATION_ERROR,
95 RtpEndUserState.CONNECTIVITY_ERROR,
96 RtpEndUserState.SECURITY_ERROR
97 );
98 private static final List<RtpEndUserState> STATES_SHOWING_SWITCH_TO_CHAT = Arrays.asList(
99 RtpEndUserState.CONNECTING,
100 RtpEndUserState.CONNECTED,
101 RtpEndUserState.RECONNECTING
102 );
103 private static final List<RtpEndUserState> STATES_CONSIDERED_CONNECTED = Arrays.asList(
104 RtpEndUserState.CONNECTED,
105 RtpEndUserState.RECONNECTING
106 );
107 private static final List<RtpEndUserState> STATES_SHOWING_PIP_PLACEHOLDER = Arrays.asList(
108 RtpEndUserState.ACCEPTING_CALL,
109 RtpEndUserState.CONNECTING,
110 RtpEndUserState.RECONNECTING
111 );
112 private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
113 private static final int REQUEST_ACCEPT_CALL = 0x1111;
114 private WeakReference<JingleRtpConnection> rtpConnectionReference;
115
116 private ActivityRtpSessionBinding binding;
117 private PowerManager.WakeLock mProximityWakeLock;
118
119 private final Handler mHandler = new Handler();
120 private final Runnable mTickExecutor = new Runnable() {
121 @Override
122 public void run() {
123 updateCallDuration();
124 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
125 }
126 };
127
128 private static Set<Media> actionToMedia(final String action) {
129 if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
130 return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
131 } else {
132 return ImmutableSet.of(Media.AUDIO);
133 }
134 }
135
136 private static void addSink(final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) {
137 try {
138 videoTrack.addSink(surfaceViewRenderer);
139 } catch (final IllegalStateException e) {
140 Log.e(Config.LOGTAG, "possible race condition on trying to display video track. ignoring", e);
141 }
142 }
143
144 @Override
145 public void onCreate(Bundle savedInstanceState) {
146 super.onCreate(savedInstanceState);
147 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON
148 | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD
149 | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED
150 | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON);
151 this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session);
152 setSupportActionBar(binding.toolbar);
153
154 //TODO: remove this - for testing dialpad input
155 //((DialpadView)findViewById(R.id.action_dialpad)).
156
157 findViewById(R.id.dialpad_1_holder).setOnClickListener(view -> dialpadPressed(view));
158 findViewById(R.id.dialpad_2_holder).setOnClickListener(view -> dialpadPressed(view));
159 findViewById(R.id.dialpad_3_holder).setOnClickListener(view -> dialpadPressed(view));
160 findViewById(R.id.dialpad_4_holder).setOnClickListener(view -> dialpadPressed(view));
161 findViewById(R.id.dialpad_5_holder).setOnClickListener(view -> dialpadPressed(view));
162 findViewById(R.id.dialpad_6_holder).setOnClickListener(view -> dialpadPressed(view));
163 findViewById(R.id.dialpad_7_holder).setOnClickListener(view -> dialpadPressed(view));
164 findViewById(R.id.dialpad_8_holder).setOnClickListener(view -> dialpadPressed(view));
165 findViewById(R.id.dialpad_9_holder).setOnClickListener(view -> dialpadPressed(view));
166 findViewById(R.id.dialpad_0_holder).setOnClickListener(view -> dialpadPressed(view));
167 findViewById(R.id.dialpad_asterisk_holder).setOnClickListener(view -> dialpadPressed(view));
168 findViewById(R.id.dialpad_pound_holder).setOnClickListener(view -> dialpadPressed(view));
169
170 if (savedInstanceState != null) {
171 int dialpad_visibility = savedInstanceState.getInt("dialpad_visibility");
172 System.out.println("dialpad_visibility onCreate = " + dialpad_visibility);
173 findViewById(R.id.dialpad).setVisibility(dialpad_visibility);
174 }
175 }
176
177
178
179 private void dialpadPressed(View dialpadKeyHolderView) {
180 JingleRtpConnection rtpConnection = requireRtpConnection();
181 rtpConnection.applyDtmfTone(dialpadKeyHolderView.getTag().toString());
182 }
183
184 @Override
185 public boolean onCreateOptionsMenu(final Menu menu) {
186 getMenuInflater().inflate(R.menu.activity_rtp_session, menu);
187 final MenuItem help = menu.findItem(R.id.action_help);
188 final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat);
189 final MenuItem dialpad = menu.findItem(R.id.action_dialpad);
190 help.setVisible(isHelpButtonVisible());
191 gotoChat.setVisible(isSwitchToConversationVisible());
192 dialpad.setVisible(isAudioOnlyConversation());
193 return super.onCreateOptionsMenu(menu);
194 }
195
196 @Override
197 public boolean onKeyDown(final int keyCode, final KeyEvent event) {
198 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
199 if (xmppConnectionService != null) {
200 if (xmppConnectionService.getNotificationService().stopSoundAndVibration()) {
201 return true;
202 }
203 }
204 }
205 return super.onKeyDown(keyCode, event);
206 }
207
208 private boolean isHelpButtonVisible() {
209 try {
210 return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState());
211 } catch (IllegalStateException e) {
212 final Intent intent = getIntent();
213 final String state = intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null;
214 if (state != null) {
215 return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state));
216 } else {
217 return false;
218 }
219 }
220 }
221
222 private boolean isSwitchToConversationVisible() {
223 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
224 return connection != null && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState());
225 }
226
227 private boolean isAudioOnlyConversation() {
228 final JingleRtpConnection connection =
229 this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
230 return connection != null &&
231 connection.getEndUserState() == RtpEndUserState.CONNECTED &&
232 !connection.isVideoEnabled();
233 }
234
235 private void switchToConversation() {
236 final Contact contact = getWith();
237 final Conversation conversation = xmppConnectionService.findOrCreateConversation(contact.getAccount(), contact.getJid(), false, true);
238 switchToConversation(conversation);
239 }
240
241 private void toggleDialpadVisibility() {
242 if (binding.dialpad.getVisibility() == View.VISIBLE) {
243 binding.dialpad.setVisibility(View.GONE);
244 }
245 else {
246 binding.dialpad.setVisibility(View.VISIBLE);
247 }
248 }
249
250 public boolean onOptionsItemSelected(final MenuItem item) {
251 switch (item.getItemId()) {
252 case R.id.action_help:
253 launchHelpInBrowser();
254 break;
255 case R.id.action_goto_chat:
256 switchToConversation();
257 break;
258 case R.id.action_dialpad:
259 toggleDialpadVisibility();
260 break;
261 }
262 return super.onOptionsItemSelected(item);
263 }
264
265 private void launchHelpInBrowser() {
266 final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP);
267 try {
268 startActivity(intent);
269 } catch (final ActivityNotFoundException e) {
270 Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG).show();
271 }
272 }
273
274 private void endCall(View view) {
275 endCall();
276 }
277
278 private void endCall() {
279 if (this.rtpConnectionReference == null) {
280 retractSessionProposal();
281 finish();
282 } else {
283 requireRtpConnection().endCall();
284 }
285 }
286
287 private void retractSessionProposal() {
288 final Intent intent = getIntent();
289 final String action = intent.getAction();
290 final Account account = extractAccount(intent);
291 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
292 final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
293 if (!Intent.ACTION_VIEW.equals(action) || state == null || !END_CARD.contains(RtpEndUserState.valueOf(state))) {
294 resetIntent(account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction()));
295 }
296 xmppConnectionService.getJingleConnectionManager().retractSessionProposal(account, with.asBareJid());
297 }
298
299 private void rejectCall(View view) {
300 requireRtpConnection().rejectCall();
301 finish();
302 }
303
304 private void acceptCall(View view) {
305 requestPermissionsAndAcceptCall();
306 }
307
308 private void requestPermissionsAndAcceptCall() {
309 final List<String> permissions;
310 if (getMedia().contains(Media.VIDEO)) {
311 permissions = ImmutableList.of(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO);
312 } else {
313 permissions = ImmutableList.of(Manifest.permission.RECORD_AUDIO);
314 }
315 if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) {
316 putScreenInCallMode();
317 checkRecorderAndAcceptCall();
318 }
319 }
320
321 private void checkRecorderAndAcceptCall() {
322 checkMicrophoneAvailability();
323 try {
324 requireRtpConnection().acceptCall();
325 } catch (final IllegalStateException e) {
326 Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
327 }
328 }
329
330 private void checkMicrophoneAvailability() {
331 new Thread(() -> {
332 final long start = SystemClock.elapsedRealtime();
333 final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable();
334 final long stop = SystemClock.elapsedRealtime();
335 Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms");
336 if (isMicrophoneAvailable) {
337 return;
338 }
339 runOnUiThread(() -> Toast.makeText(this, R.string.microphone_unavailable, Toast.LENGTH_LONG).show());
340 }
341 ).start();
342 }
343
344 private void putScreenInCallMode() {
345 putScreenInCallMode(requireRtpConnection().getMedia());
346 }
347
348 private void putScreenInCallMode(final Set<Media> media) {
349 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
350 if (!media.contains(Media.VIDEO)) {
351 final JingleRtpConnection rtpConnection = rtpConnectionReference != null ? rtpConnectionReference.get() : null;
352 final AppRTCAudioManager audioManager = rtpConnection == null ? null : rtpConnection.getAudioManager();
353 if (audioManager == null || audioManager.getSelectedAudioDevice() == AppRTCAudioManager.AudioDevice.EARPIECE) {
354 acquireProximityWakeLock();
355 }
356 }
357 }
358
359 @SuppressLint("WakelockTimeout")
360 private void acquireProximityWakeLock() {
361 final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
362 if (powerManager == null) {
363 Log.e(Config.LOGTAG, "power manager not available");
364 return;
365 }
366 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
367 if (this.mProximityWakeLock == null) {
368 this.mProximityWakeLock = powerManager.newWakeLock(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG);
369 }
370 if (!this.mProximityWakeLock.isHeld()) {
371 Log.d(Config.LOGTAG, "acquiring proximity wake lock");
372 this.mProximityWakeLock.acquire();
373 }
374 }
375 }
376
377 private void releaseProximityWakeLock() {
378 if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) {
379 Log.d(Config.LOGTAG, "releasing proximity wake lock");
380 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
381 this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY);
382 } else {
383 this.mProximityWakeLock.release();
384 }
385 this.mProximityWakeLock = null;
386 }
387 }
388
389 private void putProximityWakeLockInProperState(final AppRTCAudioManager.AudioDevice audioDevice) {
390 if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) {
391 acquireProximityWakeLock();
392 } else {
393 releaseProximityWakeLock();
394 }
395 }
396
397 @Override
398 protected void refreshUiReal() {
399
400 }
401
402 @Override
403 public void onNewIntent(final Intent intent) {
404 Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()");
405 super.onNewIntent(intent);
406 setIntent(intent);
407 if (xmppConnectionService == null) {
408 Log.d(Config.LOGTAG, "RtpSessionActivity: background service wasn't bound in onNewIntent()");
409 return;
410 }
411 final Account account = extractAccount(intent);
412 final String action = intent.getAction();
413 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
414 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
415 if (sessionId != null) {
416 Log.d(Config.LOGTAG, "reinitializing from onNewIntent()");
417 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
418 return;
419 }
420 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
421 Log.d(Config.LOGTAG, "accepting call from onNewIntent()");
422 requestPermissionsAndAcceptCall();
423 resetIntent(intent.getExtras());
424 }
425 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
426 proposeJingleRtpSession(account, with, actionToMedia(action));
427 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
428 } else {
429 throw new IllegalStateException("received onNewIntent without sessionId");
430 }
431 }
432
433 @Override
434 void onBackendConnected() {
435 final Intent intent = getIntent();
436 final String action = intent.getAction();
437 final Account account = extractAccount(intent);
438 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
439 final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID);
440 if (sessionId != null) {
441 if (initializeActivityWithRunningRtpSession(account, with, sessionId)) {
442 return;
443 }
444 if (ACTION_ACCEPT_CALL.equals(intent.getAction())) {
445 Log.d(Config.LOGTAG, "intent action was accept");
446 requestPermissionsAndAcceptCall();
447 resetIntent(intent.getExtras());
448 }
449 } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) {
450 proposeJingleRtpSession(account, with, actionToMedia(action));
451 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
452 } else if (Intent.ACTION_VIEW.equals(action)) {
453 final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE);
454 final RtpEndUserState state = extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState);
455 if (state != null) {
456 Log.d(Config.LOGTAG, "restored last state from intent extra");
457 updateButtonConfiguration(state);
458 updateVerifiedShield(false);
459 updateStateDisplay(state);
460 updateProfilePicture(state);
461 invalidateOptionsMenu();
462 }
463 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
464 if (xmppConnectionService.getJingleConnectionManager().fireJingleRtpConnectionStateUpdates()) {
465 return;
466 }
467 if (END_CARD.contains(state) || xmppConnectionService.getJingleConnectionManager().hasMatchingProposal(account, with)) {
468 return;
469 }
470 Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing");
471 finish();
472 }
473 }
474
475 private void proposeJingleRtpSession(final Account account, final Jid with, final Set<Media> media) {
476 checkMicrophoneAvailability();
477 if (with.isBareJid()) {
478 xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media);
479 } else {
480 final String sessionId = xmppConnectionService.getJingleConnectionManager().initializeRtpSession(account, with, media);
481 initializeActivityWithRunningRtpSession(account, with, sessionId);
482 resetIntent(account, with, sessionId);
483 }
484 putScreenInCallMode(media);
485 }
486
487 @Override
488 public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
489 super.onRequestPermissionsResult(requestCode, permissions, grantResults);
490 if (PermissionUtils.allGranted(grantResults)) {
491 if (requestCode == REQUEST_ACCEPT_CALL) {
492 checkRecorderAndAcceptCall();
493 }
494 } else {
495 @StringRes int res;
496 final String firstDenied = getFirstDenied(grantResults, permissions);
497 if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) {
498 res = R.string.no_microphone_permission;
499 } else if (Manifest.permission.CAMERA.equals(firstDenied)) {
500 res = R.string.no_camera_permission;
501 } else {
502 throw new IllegalStateException("Invalid permission result request");
503 }
504 Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT).show();
505 }
506 }
507
508 @Override
509 public void onStart() {
510 super.onStart();
511 mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL);
512 this.binding.remoteVideo.setOnAspectRatioChanged(this);
513 }
514
515 @Override
516 public void onStop() {
517 mHandler.removeCallbacks(mTickExecutor);
518 binding.remoteVideo.release();
519 binding.remoteVideo.setOnAspectRatioChanged(null);
520 binding.localVideo.release();
521 final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
522 final JingleRtpConnection jingleRtpConnection = weakReference == null ? null : weakReference.get();
523 if (jingleRtpConnection != null) {
524 releaseVideoTracks(jingleRtpConnection);
525 }
526 releaseProximityWakeLock();
527 super.onStop();
528 }
529
530 private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) {
531 final Optional<VideoTrack> remoteVideo = jingleRtpConnection.getRemoteVideoTrack();
532 if (remoteVideo.isPresent()) {
533 remoteVideo.get().removeSink(binding.remoteVideo);
534 }
535 final Optional<VideoTrack> localVideo = jingleRtpConnection.getLocalVideoTrack();
536 if (localVideo.isPresent()) {
537 localVideo.get().removeSink(binding.localVideo);
538 }
539 }
540
541 @Override
542 public void onBackPressed() {
543 if (isConnected()) {
544 if (switchToPictureInPicture()) {
545 return;
546 }
547 } else {
548 endCall();
549 }
550 super.onBackPressed();
551 }
552
553 @Override
554 public void onUserLeaveHint() {
555 super.onUserLeaveHint();
556 if (switchToPictureInPicture()) {
557 return;
558 }
559 //TODO apparently this method is not getting called on Android 10 when using the task switcher
560 if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) {
561 retractSessionProposal();
562 }
563 }
564
565 private boolean isConnected() {
566 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
567 return connection != null && STATES_CONSIDERED_CONNECTED.contains(connection.getEndUserState());
568 }
569
570 private boolean switchToPictureInPicture() {
571 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) {
572 if (shouldBePictureInPicture()) {
573 startPictureInPicture();
574 return true;
575 }
576 }
577 return false;
578 }
579
580 @RequiresApi(api = Build.VERSION_CODES.O)
581 private void startPictureInPicture() {
582 try {
583 final Rational rational = this.binding.remoteVideo.getAspectRatio();
584 final Rational clippedRational = Rationals.clip(rational);
585 Log.d(Config.LOGTAG, "suggested rational " + rational + ". clipped to " + clippedRational);
586 enterPictureInPictureMode(
587 new PictureInPictureParams.Builder()
588 .setAspectRatio(clippedRational)
589 .build()
590 );
591 } catch (final IllegalStateException e) {
592 //this sometimes happens on Samsung phones (possibly when Knox is enabled)
593 Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e);
594 }
595 }
596
597 @Override
598 public void onAspectRatioChanged(final Rational rational) {
599 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) {
600 final Rational clippedRational = Rationals.clip(rational);
601 Log.d(Config.LOGTAG, "suggested rational after aspect ratio change " + rational + ". clipped to " + clippedRational);
602 setPictureInPictureParams(new PictureInPictureParams.Builder()
603 .setAspectRatio(clippedRational)
604 .build());
605 }
606 }
607
608 private boolean deviceSupportsPictureInPicture() {
609 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
610 return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE);
611 } else {
612 return false;
613 }
614 }
615
616 private boolean shouldBePictureInPicture() {
617 try {
618 final JingleRtpConnection rtpConnection = requireRtpConnection();
619 return rtpConnection.getMedia().contains(Media.VIDEO) && Arrays.asList(
620 RtpEndUserState.ACCEPTING_CALL,
621 RtpEndUserState.CONNECTING,
622 RtpEndUserState.CONNECTED
623 ).contains(rtpConnection.getEndUserState());
624 } catch (final IllegalStateException e) {
625 return false;
626 }
627 }
628
629 private boolean initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
630 final WeakReference<JingleRtpConnection> reference = xmppConnectionService.getJingleConnectionManager()
631 .findJingleRtpConnection(account, with, sessionId);
632 if (reference == null || reference.get() == null) {
633 final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = xmppConnectionService
634 .getJingleConnectionManager().getTerminalSessionState(with, sessionId);
635 if (terminatedRtpSession == null) {
636 throw new IllegalStateException("failed to initialize activity with running rtp session. session not found");
637 }
638 initializeWithTerminatedSessionState(account, with, terminatedRtpSession);
639 return true;
640 }
641 this.rtpConnectionReference = reference;
642 final RtpEndUserState currentState = requireRtpConnection().getEndUserState();
643 final boolean verified = requireRtpConnection().isVerified();
644 if (currentState == RtpEndUserState.ENDED) {
645 reference.get().throwStateTransitionException();
646 finish();
647 return true;
648 }
649 final Set<Media> media = getMedia();
650 if (currentState == RtpEndUserState.INCOMING_CALL) {
651 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
652 }
653 if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(requireRtpConnection().getState())) {
654 putScreenInCallMode();
655 }
656 binding.with.setText(getWith().getDisplayName());
657 updateVideoViews(currentState);
658 updateStateDisplay(currentState, media);
659 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState));
660 updateButtonConfiguration(currentState, media);
661 updateProfilePicture(currentState);
662 invalidateOptionsMenu();
663 return false;
664 }
665
666 private void initializeWithTerminatedSessionState(final Account account, final Jid with, final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) {
667 Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()");
668 if (terminatedRtpSession.state == RtpEndUserState.ENDED) {
669 finish();
670 return;
671 }
672 RtpEndUserState state = terminatedRtpSession.state;
673 resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media);
674 updateButtonConfiguration(state);
675 updateStateDisplay(state);
676 updateProfilePicture(state);
677 updateCallDuration();
678 updateVerifiedShield(false);
679 invalidateOptionsMenu();
680 binding.with.setText(account.getRoster().getContact(with).getDisplayName());
681 }
682
683 private void reInitializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
684 runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId));
685 resetIntent(account, with, sessionId);
686 }
687
688 private void resetIntent(final Account account, final Jid with, final String sessionId) {
689 final Intent intent = new Intent(Intent.ACTION_VIEW);
690 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
691 intent.putExtra(EXTRA_WITH, with.toEscapedString());
692 intent.putExtra(EXTRA_SESSION_ID, sessionId);
693 setIntent(intent);
694 }
695
696 private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) {
697 surfaceViewRenderer.setVisibility(View.VISIBLE);
698 try {
699 surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null);
700 } catch (final IllegalStateException e) {
701 //Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized");
702 }
703 surfaceViewRenderer.setEnableHardwareScaler(true);
704 }
705
706 private void updateStateDisplay(final RtpEndUserState state) {
707 updateStateDisplay(state, Collections.emptySet());
708 }
709
710 private void updateStateDisplay(final RtpEndUserState state, final Set<Media> media) {
711 switch (state) {
712 case INCOMING_CALL:
713 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
714 if (media.contains(Media.VIDEO)) {
715 setTitle(R.string.rtp_state_incoming_video_call);
716 } else {
717 setTitle(R.string.rtp_state_incoming_call);
718 }
719 break;
720 case CONNECTING:
721 setTitle(R.string.rtp_state_connecting);
722 break;
723 case CONNECTED:
724 setTitle(R.string.rtp_state_connected);
725 break;
726 case RECONNECTING:
727 setTitle(R.string.rtp_state_reconnecting);
728 break;
729 case ACCEPTING_CALL:
730 setTitle(R.string.rtp_state_accepting_call);
731 break;
732 case ENDING_CALL:
733 setTitle(R.string.rtp_state_ending_call);
734 break;
735 case FINDING_DEVICE:
736 setTitle(R.string.rtp_state_finding_device);
737 break;
738 case RINGING:
739 setTitle(R.string.rtp_state_ringing);
740 break;
741 case DECLINED_OR_BUSY:
742 setTitle(R.string.rtp_state_declined_or_busy);
743 break;
744 case CONNECTIVITY_ERROR:
745 setTitle(R.string.rtp_state_connectivity_error);
746 break;
747 case CONNECTIVITY_LOST_ERROR:
748 setTitle(R.string.rtp_state_connectivity_lost_error);
749 break;
750 case RETRACTED:
751 setTitle(R.string.rtp_state_retracted);
752 break;
753 case APPLICATION_ERROR:
754 setTitle(R.string.rtp_state_application_failure);
755 break;
756 case SECURITY_ERROR:
757 setTitle(R.string.rtp_state_security_error);
758 break;
759 case ENDED:
760 throw new IllegalStateException("Activity should have called finishAndReleaseWakeLock();");
761 default:
762 throw new IllegalStateException(String.format("State %s has not been handled in UI", state));
763 }
764 }
765
766 private void updateVerifiedShield(final boolean verified) {
767 if (isPictureInPicture()) {
768 this.binding.verified.setVisibility(View.GONE);
769 return;
770 }
771 this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE);
772 }
773
774 private void updateProfilePicture(final RtpEndUserState state) {
775 updateProfilePicture(state, null);
776 }
777
778 private void updateProfilePicture(final RtpEndUserState state, final Contact contact) {
779 if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) {
780 final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call);
781 if (show) {
782 binding.contactPhoto.setVisibility(View.VISIBLE);
783 if (contact == null) {
784 AvatarWorkerTask.loadAvatar(getWith(), binding.contactPhoto, R.dimen.publish_avatar_size);
785 } else {
786 AvatarWorkerTask.loadAvatar(contact, binding.contactPhoto, R.dimen.publish_avatar_size);
787 }
788 } else {
789 binding.contactPhoto.setVisibility(View.GONE);
790 }
791 } else {
792 binding.contactPhoto.setVisibility(View.GONE);
793 }
794 }
795
796 private Set<Media> getMedia() {
797 return requireRtpConnection().getMedia();
798 }
799
800 private void updateButtonConfiguration(final RtpEndUserState state) {
801 updateButtonConfiguration(state, Collections.emptySet());
802 }
803
804 @SuppressLint("RestrictedApi")
805 private void updateButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
806 if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
807 this.binding.rejectCall.setVisibility(View.INVISIBLE);
808 this.binding.endCall.setVisibility(View.INVISIBLE);
809 this.binding.acceptCall.setVisibility(View.INVISIBLE);
810 } else if (state == RtpEndUserState.INCOMING_CALL) {
811 this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call));
812 this.binding.rejectCall.setOnClickListener(this::rejectCall);
813 this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
814 this.binding.rejectCall.setVisibility(View.VISIBLE);
815 this.binding.endCall.setVisibility(View.INVISIBLE);
816 this.binding.acceptCall.setContentDescription(getString(R.string.answer_call));
817 this.binding.acceptCall.setOnClickListener(this::acceptCall);
818 this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
819 this.binding.acceptCall.setVisibility(View.VISIBLE);
820 } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
821 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
822 this.binding.rejectCall.setOnClickListener(this::exit);
823 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
824 this.binding.rejectCall.setVisibility(View.VISIBLE);
825 this.binding.endCall.setVisibility(View.INVISIBLE);
826 this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail));
827 this.binding.acceptCall.setOnClickListener(this::recordVoiceMail);
828 this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_white_24dp);
829 this.binding.acceptCall.setVisibility(View.VISIBLE);
830 } else if (asList(
831 RtpEndUserState.CONNECTIVITY_ERROR,
832 RtpEndUserState.CONNECTIVITY_LOST_ERROR,
833 RtpEndUserState.APPLICATION_ERROR,
834 RtpEndUserState.RETRACTED,
835 RtpEndUserState.SECURITY_ERROR
836 ).contains(state)) {
837 this.binding.rejectCall.setContentDescription(getString(R.string.exit));
838 this.binding.rejectCall.setOnClickListener(this::exit);
839 this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp);
840 this.binding.rejectCall.setVisibility(View.VISIBLE);
841 this.binding.endCall.setVisibility(View.INVISIBLE);
842 this.binding.acceptCall.setContentDescription(getString(R.string.try_again));
843 this.binding.acceptCall.setOnClickListener(this::retry);
844 this.binding.acceptCall.setImageResource(R.drawable.ic_replay_white_48dp);
845 this.binding.acceptCall.setVisibility(View.VISIBLE);
846 } else {
847 this.binding.rejectCall.setVisibility(View.INVISIBLE);
848 this.binding.endCall.setContentDescription(getString(R.string.hang_up));
849 this.binding.endCall.setOnClickListener(this::endCall);
850 this.binding.endCall.setImageResource(R.drawable.ic_call_end_white_48dp);
851 this.binding.endCall.setVisibility(View.VISIBLE);
852 this.binding.acceptCall.setVisibility(View.INVISIBLE);
853 }
854 updateInCallButtonConfiguration(state, media);
855 }
856
857 private boolean isPictureInPicture() {
858 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
859 return isInPictureInPictureMode();
860 } else {
861 return false;
862 }
863 }
864
865 private void updateInCallButtonConfiguration() {
866 updateInCallButtonConfiguration(requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia());
867 }
868
869 @SuppressLint("RestrictedApi")
870 private void updateInCallButtonConfiguration(final RtpEndUserState state, final Set<Media> media) {
871 if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) {
872 Preconditions.checkArgument(media.size() > 0, "Media must not be empty");
873 if (media.contains(Media.VIDEO)) {
874 final JingleRtpConnection rtpConnection = requireRtpConnection();
875 updateInCallButtonConfigurationVideo(rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable());
876 } else {
877 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
878 updateInCallButtonConfigurationSpeaker(
879 audioManager.getSelectedAudioDevice(),
880 audioManager.getAudioDevices().size()
881 );
882 this.binding.inCallActionFarRight.setVisibility(View.GONE);
883 }
884 if (media.contains(Media.AUDIO)) {
885 updateInCallButtonConfigurationMicrophone(requireRtpConnection().isMicrophoneEnabled());
886 } else {
887 this.binding.inCallActionLeft.setVisibility(View.GONE);
888 }
889 } else {
890 this.binding.inCallActionLeft.setVisibility(View.GONE);
891 this.binding.inCallActionRight.setVisibility(View.GONE);
892 this.binding.inCallActionFarRight.setVisibility(View.GONE);
893 }
894 }
895
896 @SuppressLint("RestrictedApi")
897 private void updateInCallButtonConfigurationSpeaker(final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) {
898 switch (selectedAudioDevice) {
899 case EARPIECE:
900 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_black_24dp);
901 if (numberOfChoices >= 2) {
902 this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker);
903 } else {
904 this.binding.inCallActionRight.setOnClickListener(null);
905 this.binding.inCallActionRight.setClickable(false);
906 }
907 break;
908 case WIRED_HEADSET:
909 this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_black_24dp);
910 this.binding.inCallActionRight.setOnClickListener(null);
911 this.binding.inCallActionRight.setClickable(false);
912 break;
913 case SPEAKER_PHONE:
914 this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_black_24dp);
915 if (numberOfChoices >= 2) {
916 this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece);
917 } else {
918 this.binding.inCallActionRight.setOnClickListener(null);
919 this.binding.inCallActionRight.setClickable(false);
920 }
921 break;
922 case BLUETOOTH:
923 this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_black_24dp);
924 this.binding.inCallActionRight.setOnClickListener(null);
925 this.binding.inCallActionRight.setClickable(false);
926 break;
927 }
928 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
929 }
930
931 @SuppressLint("RestrictedApi")
932 private void updateInCallButtonConfigurationVideo(final boolean videoEnabled, final boolean isCameraSwitchable) {
933 this.binding.inCallActionRight.setVisibility(View.VISIBLE);
934 if (isCameraSwitchable) {
935 this.binding.inCallActionFarRight.setImageResource(R.drawable.ic_flip_camera_android_black_24dp);
936 this.binding.inCallActionFarRight.setVisibility(View.VISIBLE);
937 this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera);
938 } else {
939 this.binding.inCallActionFarRight.setVisibility(View.GONE);
940 }
941 if (videoEnabled) {
942 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_black_24dp);
943 this.binding.inCallActionRight.setOnClickListener(this::disableVideo);
944 } else {
945 this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_black_24dp);
946 this.binding.inCallActionRight.setOnClickListener(this::enableVideo);
947 }
948 }
949
950 private void switchCamera(final View view) {
951 Futures.addCallback(requireRtpConnection().switchCamera(), new FutureCallback<Boolean>() {
952 @Override
953 public void onSuccess(@NullableDecl Boolean isFrontCamera) {
954 binding.localVideo.setMirror(isFrontCamera);
955 }
956
957 @Override
958 public void onFailure(@NonNull final Throwable throwable) {
959 Log.d(Config.LOGTAG, "could not switch camera", Throwables.getRootCause(throwable));
960 Toast.makeText(RtpSessionActivity.this, R.string.could_not_switch_camera, Toast.LENGTH_LONG).show();
961 }
962 }, MainThreadExecutor.getInstance());
963 }
964
965 private void enableVideo(View view) {
966 try {
967 requireRtpConnection().setVideoEnabled(true);
968 } catch (final IllegalStateException e) {
969 Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show();
970 return;
971 }
972 updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable());
973 }
974
975 private void disableVideo(View view) {
976 requireRtpConnection().setVideoEnabled(false);
977 updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable());
978
979 }
980
981 @SuppressLint("RestrictedApi")
982 private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) {
983 if (microphoneEnabled) {
984 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_black_24dp);
985 this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone);
986 } else {
987 this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_black_24dp);
988 this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone);
989 }
990 this.binding.inCallActionLeft.setVisibility(View.VISIBLE);
991 }
992
993 private void updateCallDuration() {
994 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
995 if (connection == null || connection.getMedia().contains(Media.VIDEO)) {
996 this.binding.duration.setVisibility(View.GONE);
997 return;
998 }
999 if (connection.zeroDuration()) {
1000 this.binding.duration.setVisibility(View.GONE);
1001 } else {
1002 this.binding.duration.setText(TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false));
1003 this.binding.duration.setVisibility(View.VISIBLE);
1004 }
1005 }
1006
1007 private void updateVideoViews(final RtpEndUserState state) {
1008 if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
1009 binding.localVideo.setVisibility(View.GONE);
1010 binding.localVideo.release();
1011 binding.remoteVideoWrapper.setVisibility(View.GONE);
1012 binding.remoteVideo.release();
1013 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1014 if (isPictureInPicture()) {
1015 binding.appBarLayout.setVisibility(View.GONE);
1016 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1017 if (Arrays.asList(
1018 RtpEndUserState.APPLICATION_ERROR,
1019 RtpEndUserState.CONNECTIVITY_ERROR,
1020 RtpEndUserState.SECURITY_ERROR)
1021 .contains(state)) {
1022 binding.pipWarning.setVisibility(View.VISIBLE);
1023 binding.pipWaiting.setVisibility(View.GONE);
1024 } else {
1025 binding.pipWarning.setVisibility(View.GONE);
1026 binding.pipWaiting.setVisibility(View.GONE);
1027 }
1028 } else {
1029 binding.appBarLayout.setVisibility(View.VISIBLE);
1030 binding.pipPlaceholder.setVisibility(View.GONE);
1031 }
1032 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1033 return;
1034 }
1035 if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) {
1036 binding.localVideo.setVisibility(View.GONE);
1037 binding.remoteVideoWrapper.setVisibility(View.GONE);
1038 binding.appBarLayout.setVisibility(View.GONE);
1039 binding.pipPlaceholder.setVisibility(View.VISIBLE);
1040 binding.pipWarning.setVisibility(View.GONE);
1041 binding.pipWaiting.setVisibility(View.VISIBLE);
1042 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1043 return;
1044 }
1045 final Optional<VideoTrack> localVideoTrack = getLocalVideoTrack();
1046 if (localVideoTrack.isPresent() && !isPictureInPicture()) {
1047 ensureSurfaceViewRendererIsSetup(binding.localVideo);
1048 //paint local view over remote view
1049 binding.localVideo.setZOrderMediaOverlay(true);
1050 binding.localVideo.setMirror(requireRtpConnection().isFrontCamera());
1051 addSink(localVideoTrack.get(), binding.localVideo);
1052 } else {
1053 binding.localVideo.setVisibility(View.GONE);
1054 }
1055 final Optional<VideoTrack> remoteVideoTrack = getRemoteVideoTrack();
1056 if (remoteVideoTrack.isPresent()) {
1057 ensureSurfaceViewRendererIsSetup(binding.remoteVideo);
1058 addSink(remoteVideoTrack.get(), binding.remoteVideo);
1059 binding.remoteVideo.setScalingType(
1060 RendererCommon.ScalingType.SCALE_ASPECT_FILL,
1061 RendererCommon.ScalingType.SCALE_ASPECT_FIT
1062 );
1063 if (state == RtpEndUserState.CONNECTED) {
1064 binding.appBarLayout.setVisibility(View.GONE);
1065 getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1066 binding.remoteVideoWrapper.setVisibility(View.VISIBLE);
1067 } else {
1068 binding.appBarLayout.setVisibility(View.VISIBLE);
1069 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1070 binding.remoteVideoWrapper.setVisibility(View.GONE);
1071 }
1072 if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) {
1073 binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE);
1074 } else {
1075 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1076 }
1077 } else {
1078 getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
1079 binding.remoteVideoWrapper.setVisibility(View.GONE);
1080 binding.pipLocalMicOffIndicator.setVisibility(View.GONE);
1081 }
1082 }
1083
1084 private Optional<VideoTrack> getLocalVideoTrack() {
1085 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1086 if (connection == null) {
1087 return Optional.absent();
1088 }
1089 return connection.getLocalVideoTrack();
1090 }
1091
1092 private Optional<VideoTrack> getRemoteVideoTrack() {
1093 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1094 if (connection == null) {
1095 return Optional.absent();
1096 }
1097 return connection.getRemoteVideoTrack();
1098 }
1099
1100 private void disableMicrophone(View view) {
1101 final JingleRtpConnection rtpConnection = requireRtpConnection();
1102 if (rtpConnection.setMicrophoneEnabled(false)) {
1103 updateInCallButtonConfiguration();
1104 }
1105 }
1106
1107 private void enableMicrophone(View view) {
1108 final JingleRtpConnection rtpConnection = requireRtpConnection();
1109 if (rtpConnection.setMicrophoneEnabled(true)) {
1110 updateInCallButtonConfiguration();
1111 }
1112 }
1113
1114 private void switchToEarpiece(View view) {
1115 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE);
1116 acquireProximityWakeLock();
1117 }
1118
1119 private void switchToSpeaker(View view) {
1120 requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE);
1121 releaseProximityWakeLock();
1122 }
1123
1124 private void retry(View view) {
1125 final Intent intent = getIntent();
1126 final Account account = extractAccount(intent);
1127 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1128 final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION);
1129 final String action = intent.getAction();
1130 final Set<Media> media = actionToMedia(lastAction == null ? action : lastAction);
1131 this.rtpConnectionReference = null;
1132 Log.d(Config.LOGTAG, "attempting retry with " + with.toEscapedString());
1133 proposeJingleRtpSession(account, with, media);
1134 }
1135
1136 private void exit(final View view) {
1137 finish();
1138 }
1139
1140 private void recordVoiceMail(final View view) {
1141 final Intent intent = getIntent();
1142 final Account account = extractAccount(intent);
1143 final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH));
1144 final Conversation conversation = xmppConnectionService.findOrCreateConversation(account, with, false, true);
1145 final Intent launchIntent = new Intent(this, ConversationsActivity.class);
1146 launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION);
1147 launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid());
1148 launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP);
1149 launchIntent.putExtra(ConversationsActivity.EXTRA_POST_INIT_ACTION, ConversationsActivity.POST_ACTION_RECORD_VOICE);
1150 startActivity(launchIntent);
1151 finish();
1152 }
1153
1154 private Contact getWith() {
1155 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1156 final Account account = id.account;
1157 return account.getRoster().getContact(id.with);
1158 }
1159
1160 private JingleRtpConnection requireRtpConnection() {
1161 final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null;
1162 if (connection == null) {
1163 throw new IllegalStateException("No RTP connection found");
1164 }
1165 return connection;
1166 }
1167
1168 @Override
1169 public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) {
1170 Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
1171 if (END_CARD.contains(state)) {
1172 Log.d(Config.LOGTAG, "end card reached");
1173 releaseProximityWakeLock();
1174 runOnUiThread(() -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
1175 }
1176 if (with.isBareJid()) {
1177 updateRtpSessionProposalState(account, with, state);
1178 return;
1179 }
1180 if (emptyReference(this.rtpConnectionReference)) {
1181 if (END_CARD.contains(state)) {
1182 Log.d(Config.LOGTAG, "not reinitializing session");
1183 return;
1184 }
1185 //this happens when going from proposed session to actual session
1186 reInitializeActivityWithRunningRtpSession(account, with, sessionId);
1187 return;
1188 }
1189 final AbstractJingleConnection.Id id = requireRtpConnection().getId();
1190 final boolean verified = requireRtpConnection().isVerified();
1191 final Set<Media> media = getMedia();
1192 final Contact contact = getWith();
1193 if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
1194 if (state == RtpEndUserState.ENDED) {
1195 finish();
1196 return;
1197 }
1198 runOnUiThread(() -> {
1199 updateStateDisplay(state, media);
1200 updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state));
1201 updateButtonConfiguration(state, media);
1202 updateVideoViews(state);
1203 updateProfilePicture(state, contact);
1204 invalidateOptionsMenu();
1205 });
1206 if (END_CARD.contains(state)) {
1207 final JingleRtpConnection rtpConnection = requireRtpConnection();
1208 resetIntent(account, with, state, rtpConnection.getMedia());
1209 releaseVideoTracks(rtpConnection);
1210 this.rtpConnectionReference = null;
1211 }
1212 } else {
1213 Log.d(Config.LOGTAG, "received update for other rtp session");
1214 }
1215 }
1216
1217 @Override
1218 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1219 Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices);
1220 try {
1221 if (getMedia().contains(Media.VIDEO)) {
1222 Log.d(Config.LOGTAG, "nothing to do; in video mode");
1223 return;
1224 }
1225 final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
1226 if (endUserState == RtpEndUserState.CONNECTED) {
1227 final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager();
1228 updateInCallButtonConfigurationSpeaker(
1229 audioManager.getSelectedAudioDevice(),
1230 audioManager.getAudioDevices().size()
1231 );
1232 } else if (END_CARD.contains(endUserState)) {
1233 Log.d(Config.LOGTAG, "onAudioDeviceChanged() nothing to do because end card has been reached");
1234 } else {
1235 putProximityWakeLockInProperState(selectedAudioDevice);
1236 }
1237 } catch (IllegalStateException e) {
1238 Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed");
1239 }
1240 }
1241
1242 @Override
1243 protected void onSaveInstanceState(@NonNull @NotNull Bundle outState) {
1244 super.onSaveInstanceState(outState);
1245 int visibility = findViewById(R.id.action_dialpad).getVisibility();
1246 System.out.println("visibility onSave = " + visibility);
1247 outState.putInt("dialpad_visibility", visibility);
1248 }
1249
1250 private void updateRtpSessionProposalState(final Account account, final Jid with, final RtpEndUserState state) {
1251 final Intent currentIntent = getIntent();
1252 final String withExtra = currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH);
1253 if (withExtra == null) {
1254 return;
1255 }
1256 if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) {
1257 runOnUiThread(() -> {
1258 updateVerifiedShield(false);
1259 updateStateDisplay(state);
1260 updateButtonConfiguration(state);
1261 updateProfilePicture(state);
1262 invalidateOptionsMenu();
1263 });
1264 resetIntent(account, with, state, actionToMedia(currentIntent.getAction()));
1265 }
1266 }
1267
1268 private void resetIntent(final Bundle extras) {
1269 final Intent intent = new Intent(Intent.ACTION_VIEW);
1270 intent.putExtras(extras);
1271 setIntent(intent);
1272 }
1273
1274 private void resetIntent(final Account account, Jid with, final RtpEndUserState state, final Set<Media> media) {
1275 final Intent intent = new Intent(Intent.ACTION_VIEW);
1276 intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString());
1277 if (account.getRoster().getContact(with).getPresences().anySupport(Namespace.JINGLE_MESSAGE)) {
1278 intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString());
1279 } else {
1280 intent.putExtra(EXTRA_WITH, with.toEscapedString());
1281 }
1282 intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString());
1283 intent.putExtra(EXTRA_LAST_ACTION, media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL);
1284 setIntent(intent);
1285 }
1286
1287 private static boolean emptyReference(final WeakReference<?> weakReference) {
1288 return weakReference == null || weakReference.get() == null;
1289 }
1290}