1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.collect.ImmutableSet;
12import com.google.common.collect.Iterables;
13import com.google.common.util.concurrent.Futures;
14import com.google.common.util.concurrent.ListenableFuture;
15import com.google.common.util.concurrent.MoreExecutors;
16import com.google.common.util.concurrent.SettableFuture;
17
18import org.webrtc.AudioSource;
19import org.webrtc.AudioTrack;
20import org.webrtc.Camera1Enumerator;
21import org.webrtc.Camera2Enumerator;
22import org.webrtc.CameraEnumerationAndroid;
23import org.webrtc.CameraEnumerator;
24import org.webrtc.CameraVideoCapturer;
25import org.webrtc.CandidatePairChangeEvent;
26import org.webrtc.DataChannel;
27import org.webrtc.DefaultVideoDecoderFactory;
28import org.webrtc.DefaultVideoEncoderFactory;
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.MediaConstraints;
32import org.webrtc.MediaStream;
33import org.webrtc.MediaStreamTrack;
34import org.webrtc.PeerConnection;
35import org.webrtc.PeerConnectionFactory;
36import org.webrtc.RtpReceiver;
37import org.webrtc.RtpTransceiver;
38import org.webrtc.SdpObserver;
39import org.webrtc.SessionDescription;
40import org.webrtc.SurfaceTextureHelper;
41import org.webrtc.VideoSource;
42import org.webrtc.VideoTrack;
43import org.webrtc.audio.JavaAudioDeviceModule;
44import org.webrtc.voiceengine.WebRtcAudioEffects;
45
46import java.util.ArrayList;
47import java.util.Collections;
48import java.util.List;
49import java.util.Set;
50
51import javax.annotation.Nonnull;
52import javax.annotation.Nullable;
53
54import eu.siacs.conversations.Config;
55import eu.siacs.conversations.services.AppRTCAudioManager;
56import eu.siacs.conversations.services.XmppConnectionService;
57
58public class WebRTCWrapper {
59
60 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
61
62 //we should probably keep this in sync with: https://github.com/signalapp/Signal-Android/blob/master/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java#L296
63 private static final Set<String> HARDWARE_AEC_BLACKLIST = new ImmutableSet.Builder<String>()
64 .add("Pixel")
65 .add("Pixel XL")
66 .add("Moto G5")
67 .add("Moto G (5S) Plus")
68 .add("Moto G4")
69 .add("TA-1053")
70 .add("Mi A1")
71 .add("Mi A2")
72 .add("E5823") // Sony z5 compact
73 .add("Redmi Note 5")
74 .add("FP2") // Fairphone FP2
75 .add("MI 5")
76 .build();
77
78 private static final int CAPTURING_RESOLUTION = 1920;
79 private static final int CAPTURING_MAX_FRAME_RATE = 30;
80
81 private final EventCallback eventCallback;
82 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
83 @Override
84 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
85 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
86 }
87 };
88 private final Handler mainHandler = new Handler(Looper.getMainLooper());
89 private VideoTrack localVideoTrack = null;
90 private VideoTrack remoteVideoTrack = null;
91 private PeerConnection.PeerConnectionState currentState;
92 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
93 @Override
94 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
95 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
96 //this is called after removeTrack or addTrack
97 //and should then trigger a content-add or content-remove or something
98 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
99 }
100
101 @Override
102 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
103 eventCallback.onConnectionChange(currentState, newState);
104 currentState = newState;
105 }
106
107 @Override
108 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
109
110 }
111
112 @Override
113 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
114 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
115 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
116 }
117
118 @Override
119 public void onIceConnectionReceivingChange(boolean b) {
120
121 }
122
123 @Override
124 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
125 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
126 }
127
128 @Override
129 public void onIceCandidate(IceCandidate iceCandidate) {
130 eventCallback.onIceCandidate(iceCandidate);
131 }
132
133 @Override
134 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
135
136 }
137
138 @Override
139 public void onAddStream(MediaStream mediaStream) {
140 Log.d(EXTENDED_LOGGING_TAG, "onAddStream(numAudioTracks=" + mediaStream.audioTracks.size() + ",numVideoTracks=" + mediaStream.videoTracks.size() + ")");
141 }
142
143 @Override
144 public void onRemoveStream(MediaStream mediaStream) {
145
146 }
147
148 @Override
149 public void onDataChannel(DataChannel dataChannel) {
150
151 }
152
153 @Override
154 public void onRenegotiationNeeded() {
155 Log.d(EXTENDED_LOGGING_TAG,"onRenegotiationNeeded - current state: "+currentState);
156 }
157
158 @Override
159 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
160 final MediaStreamTrack track = rtpReceiver.track();
161 Log.d(EXTENDED_LOGGING_TAG, "onAddTrack(kind=" + (track == null ? "null" : track.kind()) + ",numMediaStreams=" + mediaStreams.length + ")");
162 if (track instanceof VideoTrack) {
163 remoteVideoTrack = (VideoTrack) track;
164 }
165 }
166
167 @Override
168 public void onTrack(RtpTransceiver transceiver) {
169 Log.d(EXTENDED_LOGGING_TAG, "onTrack(mid=" + transceiver.getMid() + ",media=" + transceiver.getMediaType() + ")");
170 }
171 };
172 @Nullable
173 private PeerConnection peerConnection = null;
174 private AudioTrack localAudioTrack = null;
175 private AppRTCAudioManager appRTCAudioManager = null;
176 private ToneManager toneManager = null;
177 private Context context = null;
178 private EglBase eglBase = null;
179 private CapturerChoice capturerChoice;
180
181 WebRTCWrapper(final EventCallback eventCallback) {
182 this.eventCallback = eventCallback;
183 }
184
185 private static void dispose(final PeerConnection peerConnection) {
186 try {
187 peerConnection.dispose();
188 } catch (final IllegalStateException e) {
189 Log.e(Config.LOGTAG, "unable to dispose of peer connection", e);
190 }
191 }
192
193 @Nullable
194 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName, Set<String> availableCameras) {
195 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
196 if (capturer == null) {
197 return null;
198 }
199 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
200 Collections.sort(choices, (a, b) -> b.width - a.width);
201 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
202 if (captureFormat.width <= CAPTURING_RESOLUTION) {
203 return new CapturerChoice(capturer, captureFormat, availableCameras);
204 }
205 }
206 return null;
207 }
208
209 private static boolean isFrontFacing(final CameraEnumerator cameraEnumerator, final String deviceName) {
210 try {
211 return cameraEnumerator.isFrontFacing(deviceName);
212 } catch (final NullPointerException e) {
213 return false;
214 }
215 }
216
217 public void setup(final XmppConnectionService service, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) throws InitializationException {
218 try {
219 PeerConnectionFactory.initialize(
220 PeerConnectionFactory.InitializationOptions.builder(service).createInitializationOptions()
221 );
222 } catch (final UnsatisfiedLinkError e) {
223 throw new InitializationException("Unable to initialize PeerConnectionFactory", e);
224 }
225 try {
226 this.eglBase = EglBase.create();
227 } catch (final RuntimeException e) {
228 throw new InitializationException("Unable to create EGL base", e);
229 }
230 this.context = service;
231 this.toneManager = service.getJingleConnectionManager().toneManager;
232 mainHandler.post(() -> {
233 appRTCAudioManager = AppRTCAudioManager.create(service, speakerPhonePreference);
234 toneManager.setAppRtcAudioManagerHasControl(true);
235 appRTCAudioManager.start(audioManagerEvents);
236 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
237 });
238 }
239
240 synchronized void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
241 Preconditions.checkState(this.eglBase != null);
242 Preconditions.checkNotNull(media);
243 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
244 final boolean setUseHardwareAcousticEchoCanceler = WebRtcAudioEffects.canUseAcousticEchoCanceler() && !HARDWARE_AEC_BLACKLIST.contains(Build.MODEL);
245 Log.d(Config.LOGTAG, String.format("setUseHardwareAcousticEchoCanceler(%s) model=%s", setUseHardwareAcousticEchoCanceler, Build.MODEL));
246 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
247 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
248 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
249 .setAudioDeviceModule(JavaAudioDeviceModule.builder(context)
250 .setUseHardwareAcousticEchoCanceler(setUseHardwareAcousticEchoCanceler)
251 .createAudioDeviceModule()
252 )
253 .createPeerConnectionFactory();
254
255
256 final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
257 rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp
258 rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
259 rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
260 rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
261 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
262 if (peerConnection == null) {
263 throw new InitializationException("Unable to create PeerConnection");
264 }
265
266 this.currentState = peerConnection.connectionState();
267
268 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
269
270 if (optionalCapturerChoice.isPresent()) {
271 this.capturerChoice = optionalCapturerChoice.get();
272 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
273 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
274 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
275 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
276 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
277 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
278
279 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
280
281 peerConnection.addTrack(this.localVideoTrack);
282 }
283
284
285 if (media.contains(Media.AUDIO)) {
286 //set up audio track
287 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
288 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
289 peerConnection.addTrack(this.localAudioTrack);
290 }
291 peerConnection.setAudioPlayout(true);
292 peerConnection.setAudioRecording(true);
293 this.peerConnection = peerConnection;
294 }
295
296 synchronized void close() {
297 final PeerConnection peerConnection = this.peerConnection;
298 final CapturerChoice capturerChoice = this.capturerChoice;
299 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
300 final EglBase eglBase = this.eglBase;
301 if (peerConnection != null) {
302 dispose(peerConnection);
303 this.peerConnection = null;
304 }
305 if (audioManager != null) {
306 toneManager.setAppRtcAudioManagerHasControl(false);
307 mainHandler.post(audioManager::stop);
308 }
309 this.localVideoTrack = null;
310 this.remoteVideoTrack = null;
311 if (capturerChoice != null) {
312 try {
313 capturerChoice.cameraVideoCapturer.stopCapture();
314 } catch (InterruptedException e) {
315 Log.e(Config.LOGTAG, "unable to stop capturing");
316 }
317 }
318 if (eglBase != null) {
319 eglBase.release();
320 this.eglBase = null;
321 }
322 }
323
324 synchronized void verifyClosed() {
325 if (this.peerConnection != null
326 || this.eglBase != null
327 || this.localVideoTrack != null
328 || this.remoteVideoTrack != null) {
329 final IllegalStateException e = new IllegalStateException("WebRTCWrapper hasn't been closed properly");
330 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
331 throw e;
332 }
333 }
334
335 boolean isCameraSwitchable() {
336 final CapturerChoice capturerChoice = this.capturerChoice;
337 return capturerChoice != null && capturerChoice.availableCameras.size() > 1;
338 }
339
340 boolean isFrontCamera() {
341 final CapturerChoice capturerChoice = this.capturerChoice;
342 return capturerChoice == null || capturerChoice.isFrontCamera;
343 }
344
345 ListenableFuture<Boolean> switchCamera() {
346 final CapturerChoice capturerChoice = this.capturerChoice;
347 if (capturerChoice == null) {
348 return Futures.immediateFailedFuture(new IllegalStateException("CameraCapturer has not been initialized"));
349 }
350 final SettableFuture<Boolean> future = SettableFuture.create();
351 capturerChoice.cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() {
352 @Override
353 public void onCameraSwitchDone(boolean isFrontCamera) {
354 capturerChoice.isFrontCamera = isFrontCamera;
355 future.set(isFrontCamera);
356 }
357
358 @Override
359 public void onCameraSwitchError(final String message) {
360 future.setException(new IllegalStateException(String.format("Unable to switch camera %s", message)));
361 }
362 });
363 return future;
364 }
365
366 boolean isMicrophoneEnabled() {
367 final AudioTrack audioTrack = this.localAudioTrack;
368 if (audioTrack == null) {
369 throw new IllegalStateException("Local audio track does not exist (yet)");
370 }
371 try {
372 return audioTrack.enabled();
373 } catch (final IllegalStateException e) {
374 //sometimes UI might still be rendering the buttons when a background thread has already ended the call
375 return false;
376 }
377 }
378
379 boolean setMicrophoneEnabled(final boolean enabled) {
380 final AudioTrack audioTrack = this.localAudioTrack;
381 if (audioTrack == null) {
382 throw new IllegalStateException("Local audio track does not exist (yet)");
383 }
384 try {
385 audioTrack.setEnabled(enabled);
386 return true;
387 } catch (final IllegalStateException e) {
388 Log.d(Config.LOGTAG, "unable to toggle microphone", e);
389 //ignoring race condition in case MediaStreamTrack has been disposed
390 return false;
391 }
392 }
393
394 boolean isVideoEnabled() {
395 final VideoTrack videoTrack = this.localVideoTrack;
396 if (videoTrack == null) {
397 return false;
398 }
399 return videoTrack.enabled();
400 }
401
402 void setVideoEnabled(final boolean enabled) {
403 final VideoTrack videoTrack = this.localVideoTrack;
404 if (videoTrack == null) {
405 throw new IllegalStateException("Local video track does not exist");
406 }
407 videoTrack.setEnabled(enabled);
408 }
409
410 ListenableFuture<SessionDescription> setLocalDescription() {
411 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
412 final SettableFuture<SessionDescription> future = SettableFuture.create();
413 peerConnection.setLocalDescription(new SetSdpObserver() {
414 @Override
415 public void onSetSuccess() {
416 final SessionDescription description = peerConnection.getLocalDescription();
417 Log.d(EXTENDED_LOGGING_TAG, "set local description:");
418 logDescription(description);
419 future.set(description);
420 }
421
422 @Override
423 public void onSetFailure(final String message) {
424 future.setException(new FailureToSetDescriptionException(message));
425 }
426 });
427 return future;
428 }, MoreExecutors.directExecutor());
429 }
430
431 private static void logDescription(final SessionDescription sessionDescription) {
432 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
433 Log.d(EXTENDED_LOGGING_TAG, line);
434 }
435 }
436
437 ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
438 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
439 logDescription(sessionDescription);
440 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
441 final SettableFuture<Void> future = SettableFuture.create();
442 peerConnection.setRemoteDescription(new SetSdpObserver() {
443 @Override
444 public void onSetSuccess() {
445 future.set(null);
446 }
447
448 @Override
449 public void onSetFailure(final String message) {
450 future.setException(new FailureToSetDescriptionException(message));
451 }
452 }, sessionDescription);
453 return future;
454 }, MoreExecutors.directExecutor());
455 }
456
457 @Nonnull
458 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
459 final PeerConnection peerConnection = this.peerConnection;
460 if (peerConnection == null) {
461 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
462 } else {
463 return Futures.immediateFuture(peerConnection);
464 }
465 }
466
467 void addIceCandidate(IceCandidate iceCandidate) {
468 requirePeerConnection().addIceCandidate(iceCandidate);
469 }
470
471 private CameraEnumerator getCameraEnumerator() {
472 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
473 return new Camera2Enumerator(requireContext());
474 } else {
475 return new Camera1Enumerator();
476 }
477 }
478
479 private Optional<CapturerChoice> getVideoCapturer() {
480 final CameraEnumerator enumerator = getCameraEnumerator();
481 final Set<String> deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames());
482 for (final String deviceName : deviceNames) {
483 if (isFrontFacing(enumerator, deviceName)) {
484 final CapturerChoice capturerChoice = of(enumerator, deviceName, deviceNames);
485 if (capturerChoice == null) {
486 return Optional.absent();
487 }
488 capturerChoice.isFrontCamera = true;
489 return Optional.of(capturerChoice);
490 }
491 }
492 if (deviceNames.size() == 0) {
493 return Optional.absent();
494 } else {
495 return Optional.fromNullable(of(enumerator, Iterables.get(deviceNames, 0), deviceNames));
496 }
497 }
498
499 public PeerConnection.PeerConnectionState getState() {
500 return requirePeerConnection().connectionState();
501 }
502
503 EglBase.Context getEglBaseContext() {
504 return this.eglBase.getEglBaseContext();
505 }
506
507 Optional<VideoTrack> getLocalVideoTrack() {
508 return Optional.fromNullable(this.localVideoTrack);
509 }
510
511 Optional<VideoTrack> getRemoteVideoTrack() {
512 return Optional.fromNullable(this.remoteVideoTrack);
513 }
514
515 private PeerConnection requirePeerConnection() {
516 final PeerConnection peerConnection = this.peerConnection;
517 if (peerConnection == null) {
518 throw new PeerConnectionNotInitialized();
519 }
520 return peerConnection;
521 }
522
523 private Context requireContext() {
524 final Context context = this.context;
525 if (context == null) {
526 throw new IllegalStateException("call setup first");
527 }
528 return context;
529 }
530
531 AppRTCAudioManager getAudioManager() {
532 return appRTCAudioManager;
533 }
534
535 public interface EventCallback {
536 void onIceCandidate(IceCandidate iceCandidate);
537
538 void onConnectionChange(PeerConnection.PeerConnectionState oldState, PeerConnection.PeerConnectionState newState);
539
540 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
541 }
542
543 private static abstract class SetSdpObserver implements SdpObserver {
544
545 @Override
546 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
547 throw new IllegalStateException("Not able to use SetSdpObserver");
548 }
549
550 @Override
551 public void onCreateFailure(String s) {
552 throw new IllegalStateException("Not able to use SetSdpObserver");
553 }
554
555 }
556
557 private static abstract class CreateSdpObserver implements SdpObserver {
558
559
560 @Override
561 public void onSetSuccess() {
562 throw new IllegalStateException("Not able to use CreateSdpObserver");
563 }
564
565
566 @Override
567 public void onSetFailure(String s) {
568 throw new IllegalStateException("Not able to use CreateSdpObserver");
569 }
570 }
571
572 static class InitializationException extends Exception {
573
574 private InitializationException(final String message, final Throwable throwable) {
575 super(message, throwable);
576 }
577
578 private InitializationException(final String message) {
579 super(message);
580 }
581 }
582
583 public static class PeerConnectionNotInitialized extends IllegalStateException {
584
585 private PeerConnectionNotInitialized() {
586 super("initialize PeerConnection first");
587 }
588
589 }
590
591 private static class FailureToSetDescriptionException extends IllegalArgumentException {
592 public FailureToSetDescriptionException(String message) {
593 super(message);
594 }
595 }
596
597 private static class CapturerChoice {
598 private final CameraVideoCapturer cameraVideoCapturer;
599 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
600 private final Set<String> availableCameras;
601 private boolean isFrontCamera = false;
602
603 CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat, Set<String> cameras) {
604 this.cameraVideoCapturer = cameraVideoCapturer;
605 this.captureFormat = captureFormat;
606 this.availableCameras = cameras;
607 }
608
609 int getFrameRate() {
610 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
611 }
612 }
613}