1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.collect.ImmutableSet;
12import com.google.common.collect.Iterables;
13import com.google.common.util.concurrent.Futures;
14import com.google.common.util.concurrent.ListenableFuture;
15import com.google.common.util.concurrent.MoreExecutors;
16import com.google.common.util.concurrent.SettableFuture;
17
18import org.webrtc.AudioSource;
19import org.webrtc.AudioTrack;
20import org.webrtc.Camera1Enumerator;
21import org.webrtc.Camera2Enumerator;
22import org.webrtc.CameraEnumerationAndroid;
23import org.webrtc.CameraEnumerator;
24import org.webrtc.CameraVideoCapturer;
25import org.webrtc.CandidatePairChangeEvent;
26import org.webrtc.DataChannel;
27import org.webrtc.DefaultVideoDecoderFactory;
28import org.webrtc.DefaultVideoEncoderFactory;
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.MediaConstraints;
32import org.webrtc.MediaStream;
33import org.webrtc.MediaStreamTrack;
34import org.webrtc.PeerConnection;
35import org.webrtc.PeerConnectionFactory;
36import org.webrtc.RtpReceiver;
37import org.webrtc.RtpTransceiver;
38import org.webrtc.SdpObserver;
39import org.webrtc.SessionDescription;
40import org.webrtc.SurfaceTextureHelper;
41import org.webrtc.VideoSource;
42import org.webrtc.VideoTrack;
43import org.webrtc.audio.JavaAudioDeviceModule;
44import org.webrtc.voiceengine.WebRtcAudioEffects;
45
46import java.util.ArrayList;
47import java.util.Collections;
48import java.util.List;
49import java.util.Set;
50
51import javax.annotation.Nonnull;
52import javax.annotation.Nullable;
53
54import eu.siacs.conversations.Config;
55import eu.siacs.conversations.services.AppRTCAudioManager;
56import eu.siacs.conversations.services.XmppConnectionService;
57
58public class WebRTCWrapper {
59
60 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
61
62 //we should probably keep this in sync with: https://github.com/signalapp/Signal-Android/blob/master/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java#L296
63 private static final Set<String> HARDWARE_AEC_BLACKLIST = new ImmutableSet.Builder<String>()
64 .add("Pixel")
65 .add("Pixel XL")
66 .add("Moto G5")
67 .add("Moto G (5S) Plus")
68 .add("Moto G4")
69 .add("TA-1053")
70 .add("Mi A1")
71 .add("Mi A2")
72 .add("E5823") // Sony z5 compact
73 .add("Redmi Note 5")
74 .add("FP2") // Fairphone FP2
75 .add("MI 5")
76 .build();
77
78 private static final int CAPTURING_RESOLUTION = 1920;
79 private static final int CAPTURING_MAX_FRAME_RATE = 30;
80
81 private final EventCallback eventCallback;
82 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
83 @Override
84 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
85 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
86 }
87 };
88 private final Handler mainHandler = new Handler(Looper.getMainLooper());
89 private VideoTrack localVideoTrack = null;
90 private VideoTrack remoteVideoTrack = null;
91 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
92 @Override
93 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
94 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
95 //this is called after removeTrack or addTrack
96 //and should then trigger a content-add or content-remove or something
97 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
98 }
99
100 @Override
101 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
102 eventCallback.onConnectionChange(newState);
103 }
104
105 @Override
106 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
107
108 }
109
110 @Override
111 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
112 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
113 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
114 }
115
116 @Override
117 public void onIceConnectionReceivingChange(boolean b) {
118
119 }
120
121 @Override
122 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
123 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
124 }
125
126 @Override
127 public void onIceCandidate(IceCandidate iceCandidate) {
128 eventCallback.onIceCandidate(iceCandidate);
129 }
130
131 @Override
132 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
133
134 }
135
136 @Override
137 public void onAddStream(MediaStream mediaStream) {
138 Log.d(EXTENDED_LOGGING_TAG, "onAddStream(numAudioTracks=" + mediaStream.audioTracks.size() + ",numVideoTracks=" + mediaStream.videoTracks.size() + ")");
139 }
140
141 @Override
142 public void onRemoveStream(MediaStream mediaStream) {
143
144 }
145
146 @Override
147 public void onDataChannel(DataChannel dataChannel) {
148
149 }
150
151 @Override
152 public void onRenegotiationNeeded() {
153
154 }
155
156 @Override
157 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
158 final MediaStreamTrack track = rtpReceiver.track();
159 Log.d(EXTENDED_LOGGING_TAG, "onAddTrack(kind=" + (track == null ? "null" : track.kind()) + ",numMediaStreams=" + mediaStreams.length + ")");
160 if (track instanceof VideoTrack) {
161 remoteVideoTrack = (VideoTrack) track;
162 }
163 }
164
165 @Override
166 public void onTrack(RtpTransceiver transceiver) {
167 Log.d(EXTENDED_LOGGING_TAG, "onTrack(mid=" + transceiver.getMid() + ",media=" + transceiver.getMediaType() + ")");
168 }
169 };
170 @Nullable
171 private PeerConnection peerConnection = null;
172 private AudioTrack localAudioTrack = null;
173 private AppRTCAudioManager appRTCAudioManager = null;
174 private ToneManager toneManager = null;
175 private Context context = null;
176 private EglBase eglBase = null;
177 private CapturerChoice capturerChoice;
178
179 WebRTCWrapper(final EventCallback eventCallback) {
180 this.eventCallback = eventCallback;
181 }
182
183 private static void dispose(final PeerConnection peerConnection) {
184 try {
185 peerConnection.dispose();
186 } catch (final IllegalStateException e) {
187 Log.e(Config.LOGTAG, "unable to dispose of peer connection", e);
188 }
189 }
190
191 @Nullable
192 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName, Set<String> availableCameras) {
193 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
194 if (capturer == null) {
195 return null;
196 }
197 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
198 Collections.sort(choices, (a, b) -> b.width - a.width);
199 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
200 if (captureFormat.width <= CAPTURING_RESOLUTION) {
201 return new CapturerChoice(capturer, captureFormat, availableCameras);
202 }
203 }
204 return null;
205 }
206
207 private static boolean isFrontFacing(final CameraEnumerator cameraEnumerator, final String deviceName) {
208 try {
209 return cameraEnumerator.isFrontFacing(deviceName);
210 } catch (final NullPointerException e) {
211 return false;
212 }
213 }
214
215 public void setup(final XmppConnectionService service, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) throws InitializationException {
216 try {
217 PeerConnectionFactory.initialize(
218 PeerConnectionFactory.InitializationOptions.builder(service).createInitializationOptions()
219 );
220 } catch (final UnsatisfiedLinkError e) {
221 throw new InitializationException("Unable to initialize PeerConnectionFactory", e);
222 }
223 try {
224 this.eglBase = EglBase.create();
225 } catch (final RuntimeException e) {
226 throw new InitializationException("Unable to create EGL base", e);
227 }
228 this.context = service;
229 this.toneManager = service.getJingleConnectionManager().toneManager;
230 mainHandler.post(() -> {
231 appRTCAudioManager = AppRTCAudioManager.create(service, speakerPhonePreference);
232 toneManager.setAppRtcAudioManagerHasControl(true);
233 appRTCAudioManager.start(audioManagerEvents);
234 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
235 });
236 }
237
238 synchronized void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
239 Preconditions.checkState(this.eglBase != null);
240 Preconditions.checkNotNull(media);
241 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
242 final boolean setUseHardwareAcousticEchoCanceler = WebRtcAudioEffects.canUseAcousticEchoCanceler() && !HARDWARE_AEC_BLACKLIST.contains(Build.MODEL);
243 Log.d(Config.LOGTAG, String.format("setUseHardwareAcousticEchoCanceler(%s) model=%s", setUseHardwareAcousticEchoCanceler, Build.MODEL));
244 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
245 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
246 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
247 .setAudioDeviceModule(JavaAudioDeviceModule.builder(context)
248 .setUseHardwareAcousticEchoCanceler(setUseHardwareAcousticEchoCanceler)
249 .createAudioDeviceModule()
250 )
251 .createPeerConnectionFactory();
252
253
254 final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
255 rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp
256 rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
257 rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
258 rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
259 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
260 if (peerConnection == null) {
261 throw new InitializationException("Unable to create PeerConnection");
262 }
263
264 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
265
266 if (optionalCapturerChoice.isPresent()) {
267 this.capturerChoice = optionalCapturerChoice.get();
268 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
269 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
270 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
271 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
272 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
273 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
274
275 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
276
277 peerConnection.addTrack(this.localVideoTrack);
278 }
279
280
281 if (media.contains(Media.AUDIO)) {
282 //set up audio track
283 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
284 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
285 peerConnection.addTrack(this.localAudioTrack);
286 }
287 peerConnection.setAudioPlayout(true);
288 peerConnection.setAudioRecording(true);
289 this.peerConnection = peerConnection;
290 }
291
292 synchronized void close() {
293 final PeerConnection peerConnection = this.peerConnection;
294 final CapturerChoice capturerChoice = this.capturerChoice;
295 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
296 final EglBase eglBase = this.eglBase;
297 if (peerConnection != null) {
298 dispose(peerConnection);
299 this.peerConnection = null;
300 }
301 if (audioManager != null) {
302 toneManager.setAppRtcAudioManagerHasControl(false);
303 mainHandler.post(audioManager::stop);
304 }
305 this.localVideoTrack = null;
306 this.remoteVideoTrack = null;
307 if (capturerChoice != null) {
308 try {
309 capturerChoice.cameraVideoCapturer.stopCapture();
310 } catch (InterruptedException e) {
311 Log.e(Config.LOGTAG, "unable to stop capturing");
312 }
313 }
314 if (eglBase != null) {
315 eglBase.release();
316 this.eglBase = null;
317 }
318 }
319
320 synchronized void verifyClosed() {
321 if (this.peerConnection != null
322 || this.eglBase != null
323 || this.localVideoTrack != null
324 || this.remoteVideoTrack != null) {
325 final IllegalStateException e = new IllegalStateException("WebRTCWrapper hasn't been closed properly");
326 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
327 throw e;
328 }
329 }
330
331 boolean isCameraSwitchable() {
332 final CapturerChoice capturerChoice = this.capturerChoice;
333 return capturerChoice != null && capturerChoice.availableCameras.size() > 1;
334 }
335
336 boolean isFrontCamera() {
337 final CapturerChoice capturerChoice = this.capturerChoice;
338 return capturerChoice == null || capturerChoice.isFrontCamera;
339 }
340
341 ListenableFuture<Boolean> switchCamera() {
342 final CapturerChoice capturerChoice = this.capturerChoice;
343 if (capturerChoice == null) {
344 return Futures.immediateFailedFuture(new IllegalStateException("CameraCapturer has not been initialized"));
345 }
346 final SettableFuture<Boolean> future = SettableFuture.create();
347 capturerChoice.cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() {
348 @Override
349 public void onCameraSwitchDone(boolean isFrontCamera) {
350 capturerChoice.isFrontCamera = isFrontCamera;
351 future.set(isFrontCamera);
352 }
353
354 @Override
355 public void onCameraSwitchError(final String message) {
356 future.setException(new IllegalStateException(String.format("Unable to switch camera %s", message)));
357 }
358 });
359 return future;
360 }
361
362 boolean isMicrophoneEnabled() {
363 final AudioTrack audioTrack = this.localAudioTrack;
364 if (audioTrack == null) {
365 throw new IllegalStateException("Local audio track does not exist (yet)");
366 }
367 try {
368 return audioTrack.enabled();
369 } catch (final IllegalStateException e) {
370 //sometimes UI might still be rendering the buttons when a background thread has already ended the call
371 return false;
372 }
373 }
374
375 boolean setMicrophoneEnabled(final boolean enabled) {
376 final AudioTrack audioTrack = this.localAudioTrack;
377 if (audioTrack == null) {
378 throw new IllegalStateException("Local audio track does not exist (yet)");
379 }
380 try {
381 audioTrack.setEnabled(enabled);
382 return true;
383 } catch (final IllegalStateException e) {
384 Log.d(Config.LOGTAG, "unable to toggle microphone", e);
385 //ignoring race condition in case MediaStreamTrack has been disposed
386 return false;
387 }
388 }
389
390 boolean isVideoEnabled() {
391 final VideoTrack videoTrack = this.localVideoTrack;
392 if (videoTrack == null) {
393 return false;
394 }
395 return videoTrack.enabled();
396 }
397
398 void setVideoEnabled(final boolean enabled) {
399 final VideoTrack videoTrack = this.localVideoTrack;
400 if (videoTrack == null) {
401 throw new IllegalStateException("Local video track does not exist");
402 }
403 videoTrack.setEnabled(enabled);
404 }
405
406 ListenableFuture<SessionDescription> setLocalDescription() {
407 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
408 final SettableFuture<SessionDescription> future = SettableFuture.create();
409 peerConnection.setLocalDescription(new SetSdpObserver() {
410 @Override
411 public void onSetSuccess() {
412 final SessionDescription description = peerConnection.getLocalDescription();
413 Log.d(EXTENDED_LOGGING_TAG, "set local description:");
414 logDescription(description);
415 future.set(description);
416 }
417
418 @Override
419 public void onSetFailure(final String message) {
420 future.setException(new FailureToSetDescriptionException(message));
421 }
422 });
423 return future;
424 }, MoreExecutors.directExecutor());
425 }
426
427 private static void logDescription(final SessionDescription sessionDescription) {
428 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
429 Log.d(EXTENDED_LOGGING_TAG, line);
430 }
431 }
432
433 ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
434 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
435 logDescription(sessionDescription);
436 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
437 final SettableFuture<Void> future = SettableFuture.create();
438 peerConnection.setRemoteDescription(new SetSdpObserver() {
439 @Override
440 public void onSetSuccess() {
441 future.set(null);
442 }
443
444 @Override
445 public void onSetFailure(final String message) {
446 future.setException(new FailureToSetDescriptionException(message));
447 }
448 }, sessionDescription);
449 return future;
450 }, MoreExecutors.directExecutor());
451 }
452
453 @Nonnull
454 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
455 final PeerConnection peerConnection = this.peerConnection;
456 if (peerConnection == null) {
457 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
458 } else {
459 return Futures.immediateFuture(peerConnection);
460 }
461 }
462
463 void addIceCandidate(IceCandidate iceCandidate) {
464 requirePeerConnection().addIceCandidate(iceCandidate);
465 }
466
467 private CameraEnumerator getCameraEnumerator() {
468 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
469 return new Camera2Enumerator(requireContext());
470 } else {
471 return new Camera1Enumerator();
472 }
473 }
474
475 private Optional<CapturerChoice> getVideoCapturer() {
476 final CameraEnumerator enumerator = getCameraEnumerator();
477 final Set<String> deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames());
478 for (final String deviceName : deviceNames) {
479 if (isFrontFacing(enumerator, deviceName)) {
480 final CapturerChoice capturerChoice = of(enumerator, deviceName, deviceNames);
481 if (capturerChoice == null) {
482 return Optional.absent();
483 }
484 capturerChoice.isFrontCamera = true;
485 return Optional.of(capturerChoice);
486 }
487 }
488 if (deviceNames.size() == 0) {
489 return Optional.absent();
490 } else {
491 return Optional.fromNullable(of(enumerator, Iterables.get(deviceNames, 0), deviceNames));
492 }
493 }
494
495 public PeerConnection.PeerConnectionState getState() {
496 return requirePeerConnection().connectionState();
497 }
498
499 EglBase.Context getEglBaseContext() {
500 return this.eglBase.getEglBaseContext();
501 }
502
503 Optional<VideoTrack> getLocalVideoTrack() {
504 return Optional.fromNullable(this.localVideoTrack);
505 }
506
507 Optional<VideoTrack> getRemoteVideoTrack() {
508 return Optional.fromNullable(this.remoteVideoTrack);
509 }
510
511 private PeerConnection requirePeerConnection() {
512 final PeerConnection peerConnection = this.peerConnection;
513 if (peerConnection == null) {
514 throw new PeerConnectionNotInitialized();
515 }
516 return peerConnection;
517 }
518
519 private Context requireContext() {
520 final Context context = this.context;
521 if (context == null) {
522 throw new IllegalStateException("call setup first");
523 }
524 return context;
525 }
526
527 AppRTCAudioManager getAudioManager() {
528 return appRTCAudioManager;
529 }
530
531 public interface EventCallback {
532 void onIceCandidate(IceCandidate iceCandidate);
533
534 void onConnectionChange(PeerConnection.PeerConnectionState newState);
535
536 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
537 }
538
539 private static abstract class SetSdpObserver implements SdpObserver {
540
541 @Override
542 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
543 throw new IllegalStateException("Not able to use SetSdpObserver");
544 }
545
546 @Override
547 public void onCreateFailure(String s) {
548 throw new IllegalStateException("Not able to use SetSdpObserver");
549 }
550
551 }
552
553 private static abstract class CreateSdpObserver implements SdpObserver {
554
555
556 @Override
557 public void onSetSuccess() {
558 throw new IllegalStateException("Not able to use CreateSdpObserver");
559 }
560
561
562 @Override
563 public void onSetFailure(String s) {
564 throw new IllegalStateException("Not able to use CreateSdpObserver");
565 }
566 }
567
568 static class InitializationException extends Exception {
569
570 private InitializationException(final String message, final Throwable throwable) {
571 super(message, throwable);
572 }
573
574 private InitializationException(final String message) {
575 super(message);
576 }
577 }
578
579 public static class PeerConnectionNotInitialized extends IllegalStateException {
580
581 private PeerConnectionNotInitialized() {
582 super("initialize PeerConnection first");
583 }
584
585 }
586
587 private static class FailureToSetDescriptionException extends IllegalArgumentException {
588 public FailureToSetDescriptionException(String message) {
589 super(message);
590 }
591 }
592
593 private static class CapturerChoice {
594 private final CameraVideoCapturer cameraVideoCapturer;
595 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
596 private final Set<String> availableCameras;
597 private boolean isFrontCamera = false;
598
599 CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat, Set<String> cameras) {
600 this.cameraVideoCapturer = cameraVideoCapturer;
601 this.captureFormat = captureFormat;
602 this.availableCameras = cameras;
603 }
604
605 int getFrameRate() {
606 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
607 }
608 }
609}