1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.collect.ImmutableSet;
12import com.google.common.util.concurrent.Futures;
13import com.google.common.util.concurrent.ListenableFuture;
14import com.google.common.util.concurrent.MoreExecutors;
15import com.google.common.util.concurrent.SettableFuture;
16
17import org.webrtc.AudioSource;
18import org.webrtc.AudioTrack;
19import org.webrtc.CandidatePairChangeEvent;
20import org.webrtc.DataChannel;
21import org.webrtc.DefaultVideoDecoderFactory;
22import org.webrtc.DefaultVideoEncoderFactory;
23import org.webrtc.EglBase;
24import org.webrtc.IceCandidate;
25import org.webrtc.MediaConstraints;
26import org.webrtc.MediaStream;
27import org.webrtc.MediaStreamTrack;
28import org.webrtc.PeerConnection;
29import org.webrtc.PeerConnectionFactory;
30import org.webrtc.RtpReceiver;
31import org.webrtc.RtpTransceiver;
32import org.webrtc.SdpObserver;
33import org.webrtc.SessionDescription;
34import org.webrtc.VideoTrack;
35import org.webrtc.audio.JavaAudioDeviceModule;
36import org.webrtc.voiceengine.WebRtcAudioEffects;
37
38import java.util.LinkedList;
39import java.util.List;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutorService;
43import java.util.concurrent.Executors;
44import java.util.concurrent.atomic.AtomicBoolean;
45
46import javax.annotation.Nonnull;
47import javax.annotation.Nullable;
48
49import eu.siacs.conversations.Config;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51import eu.siacs.conversations.services.XmppConnectionService;
52
53@SuppressWarnings("UnstableApiUsage")
54public class WebRTCWrapper {
55
56 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
57
58 private final ExecutorService executorService = Executors.newSingleThreadExecutor();
59
60 private static final Set<String> HARDWARE_AEC_BLACKLIST =
61 new ImmutableSet.Builder<String>()
62 .add("Pixel")
63 .add("Pixel XL")
64 .add("Moto G5")
65 .add("Moto G (5S) Plus")
66 .add("Moto G4")
67 .add("TA-1053")
68 .add("Mi A1")
69 .add("Mi A2")
70 .add("E5823") // Sony z5 compact
71 .add("Redmi Note 5")
72 .add("FP2") // Fairphone FP2
73 .add("MI 5")
74 .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte)
75 .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte)
76 .add("GT-I9505") // Samsung Galaxy S4 (jfltexx)
77 .build();
78
79 private final EventCallback eventCallback;
80 private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false);
81 private final Queue<IceCandidate> iceCandidates = new LinkedList<>();
82 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents =
83 new AppRTCAudioManager.AudioManagerEvents() {
84 @Override
85 public void onAudioDeviceChanged(
86 AppRTCAudioManager.AudioDevice selectedAudioDevice,
87 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
88 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
89 }
90 };
91 private final Handler mainHandler = new Handler(Looper.getMainLooper());
92 private TrackWrapper<AudioTrack> localAudioTrack = null;
93 private TrackWrapper<VideoTrack> localVideoTrack = null;
94 private VideoTrack remoteVideoTrack = null;
95 private final PeerConnection.Observer peerConnectionObserver =
96 new PeerConnection.Observer() {
97 @Override
98 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
99 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
100 // this is called after removeTrack or addTrack
101 // and should then trigger a content-add or content-remove or something
102 // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
103 }
104
105 @Override
106 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
107 eventCallback.onConnectionChange(newState);
108 }
109
110 @Override
111 public void onIceConnectionChange(
112 PeerConnection.IceConnectionState iceConnectionState) {
113 Log.d(
114 EXTENDED_LOGGING_TAG,
115 "onIceConnectionChange(" + iceConnectionState + ")");
116 }
117
118 @Override
119 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
120 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
121 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
122 }
123
124 @Override
125 public void onIceConnectionReceivingChange(boolean b) {}
126
127 @Override
128 public void onIceGatheringChange(
129 PeerConnection.IceGatheringState iceGatheringState) {
130 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
131 }
132
133 @Override
134 public void onIceCandidate(IceCandidate iceCandidate) {
135 if (readyToReceivedIceCandidates.get()) {
136 eventCallback.onIceCandidate(iceCandidate);
137 } else {
138 iceCandidates.add(iceCandidate);
139 }
140 }
141
142 @Override
143 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {}
144
145 @Override
146 public void onAddStream(MediaStream mediaStream) {
147 Log.d(
148 EXTENDED_LOGGING_TAG,
149 "onAddStream(numAudioTracks="
150 + mediaStream.audioTracks.size()
151 + ",numVideoTracks="
152 + mediaStream.videoTracks.size()
153 + ")");
154 }
155
156 @Override
157 public void onRemoveStream(MediaStream mediaStream) {}
158
159 @Override
160 public void onDataChannel(DataChannel dataChannel) {}
161
162 @Override
163 public void onRenegotiationNeeded() {
164 Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()");
165 final PeerConnection.PeerConnectionState currentState =
166 peerConnection == null ? null : peerConnection.connectionState();
167 if (currentState != null
168 && currentState != PeerConnection.PeerConnectionState.NEW) {
169 eventCallback.onRenegotiationNeeded();
170 }
171 }
172
173 @Override
174 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
175 final MediaStreamTrack track = rtpReceiver.track();
176 Log.d(
177 EXTENDED_LOGGING_TAG,
178 "onAddTrack(kind="
179 + (track == null ? "null" : track.kind())
180 + ",numMediaStreams="
181 + mediaStreams.length
182 + ")");
183 if (track instanceof VideoTrack) {
184 remoteVideoTrack = (VideoTrack) track;
185 }
186 }
187
188 @Override
189 public void onTrack(RtpTransceiver transceiver) {
190 Log.d(
191 EXTENDED_LOGGING_TAG,
192 "onTrack(mid="
193 + transceiver.getMid()
194 + ",media="
195 + transceiver.getMediaType()
196 + ")");
197 }
198 };
199 @Nullable private PeerConnection peerConnection = null;
200 private AppRTCAudioManager appRTCAudioManager = null;
201 private ToneManager toneManager = null;
202 private Context context = null;
203 private EglBase eglBase = null;
204 private VideoSourceWrapper videoSourceWrapper;
205
206 WebRTCWrapper(final EventCallback eventCallback) {
207 this.eventCallback = eventCallback;
208 }
209
210 private static void dispose(final PeerConnection peerConnection) {
211 try {
212 peerConnection.dispose();
213 } catch (final IllegalStateException e) {
214 Log.e(Config.LOGTAG, "unable to dispose of peer connection", e);
215 }
216 }
217
218 public void setup(
219 final XmppConnectionService service,
220 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference)
221 throws InitializationException {
222 try {
223 PeerConnectionFactory.initialize(
224 PeerConnectionFactory.InitializationOptions.builder(service)
225 .setFieldTrials("WebRTC-BindUsingInterfaceName/Enabled/")
226 .createInitializationOptions());
227 } catch (final UnsatisfiedLinkError e) {
228 throw new InitializationException("Unable to initialize PeerConnectionFactory", e);
229 }
230 try {
231 this.eglBase = EglBase.create();
232 } catch (final RuntimeException e) {
233 throw new InitializationException("Unable to create EGL base", e);
234 }
235 this.context = service;
236 this.toneManager = service.getJingleConnectionManager().toneManager;
237 mainHandler.post(
238 () -> {
239 appRTCAudioManager = AppRTCAudioManager.create(service, speakerPhonePreference);
240 toneManager.setAppRtcAudioManagerHasControl(true);
241 appRTCAudioManager.start(audioManagerEvents);
242 eventCallback.onAudioDeviceChanged(
243 appRTCAudioManager.getSelectedAudioDevice(),
244 appRTCAudioManager.getAudioDevices());
245 });
246 }
247
248 synchronized void initializePeerConnection(
249 final Set<Media> media, final List<PeerConnection.IceServer> iceServers)
250 throws InitializationException {
251 Preconditions.checkState(this.eglBase != null);
252 Preconditions.checkNotNull(media);
253 Preconditions.checkArgument(
254 media.size() > 0, "media can not be empty when initializing peer connection");
255 final boolean setUseHardwareAcousticEchoCanceler =
256 WebRtcAudioEffects.canUseAcousticEchoCanceler()
257 && !HARDWARE_AEC_BLACKLIST.contains(Build.MODEL);
258 Log.d(
259 Config.LOGTAG,
260 String.format(
261 "setUseHardwareAcousticEchoCanceler(%s) model=%s",
262 setUseHardwareAcousticEchoCanceler, Build.MODEL));
263 PeerConnectionFactory peerConnectionFactory =
264 PeerConnectionFactory.builder()
265 .setVideoDecoderFactory(
266 new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
267 .setVideoEncoderFactory(
268 new DefaultVideoEncoderFactory(
269 eglBase.getEglBaseContext(), true, true))
270 .setAudioDeviceModule(
271 JavaAudioDeviceModule.builder(context)
272 .setUseHardwareAcousticEchoCanceler(
273 setUseHardwareAcousticEchoCanceler)
274 .createAudioDeviceModule())
275 .createPeerConnectionFactory();
276
277 final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers);
278 final PeerConnection peerConnection =
279 peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
280 if (peerConnection == null) {
281 throw new InitializationException("Unable to create PeerConnection");
282 }
283
284 final Optional<VideoSourceWrapper> optionalVideoSourceWrapper =
285 media.contains(Media.VIDEO)
286 ? new VideoSourceWrapper.Factory(requireContext()).create()
287 : Optional.absent();
288
289 if (optionalVideoSourceWrapper.isPresent()) {
290 this.videoSourceWrapper = optionalVideoSourceWrapper.get();
291 this.videoSourceWrapper.initialize(
292 peerConnectionFactory, context, eglBase.getEglBaseContext());
293 this.videoSourceWrapper.startCapture();
294
295 final VideoTrack videoTrack =
296 peerConnectionFactory.createVideoTrack(
297 "my-video-track", this.videoSourceWrapper.getVideoSource());
298
299 this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack);
300 }
301
302 if (media.contains(Media.AUDIO)) {
303 // set up audio track
304 final AudioSource audioSource =
305 peerConnectionFactory.createAudioSource(new MediaConstraints());
306 final AudioTrack audioTrack =
307 peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
308 this.localAudioTrack = TrackWrapper.addTrack(peerConnection, audioTrack);
309 }
310 peerConnection.setAudioPlayout(true);
311 peerConnection.setAudioRecording(true);
312
313 this.peerConnection = peerConnection;
314 }
315
316 private static PeerConnection.RTCConfiguration buildConfiguration(
317 final List<PeerConnection.IceServer> iceServers) {
318 final PeerConnection.RTCConfiguration rtcConfig =
319 new PeerConnection.RTCConfiguration(iceServers);
320 rtcConfig.tcpCandidatePolicy =
321 PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp
322 rtcConfig.continualGatheringPolicy =
323 PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
324 rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
325 rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
326 rtcConfig.enableImplicitRollback = true;
327 return rtcConfig;
328 }
329
330 void reconfigurePeerConnection(final List<PeerConnection.IceServer> iceServers) {
331 requirePeerConnection().setConfiguration(buildConfiguration(iceServers));
332 }
333
334 void restartIce() {
335 executorService.execute(() -> requirePeerConnection().restartIce());
336 }
337
338 public void setIsReadyToReceiveIceCandidates(final boolean ready) {
339 readyToReceivedIceCandidates.set(ready);
340 while (ready && iceCandidates.peek() != null) {
341 eventCallback.onIceCandidate(iceCandidates.poll());
342 }
343 }
344
345 synchronized void close() {
346 final PeerConnection peerConnection = this.peerConnection;
347 final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
348 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
349 final EglBase eglBase = this.eglBase;
350 if (peerConnection != null) {
351 dispose(peerConnection);
352 this.peerConnection = null;
353 }
354 if (audioManager != null) {
355 toneManager.setAppRtcAudioManagerHasControl(false);
356 mainHandler.post(audioManager::stop);
357 }
358 this.localVideoTrack = null;
359 this.remoteVideoTrack = null;
360 if (videoSourceWrapper != null) {
361 try {
362 videoSourceWrapper.stopCapture();
363 } catch (final InterruptedException e) {
364 Log.e(Config.LOGTAG, "unable to stop capturing");
365 }
366 // TODO call dispose
367 }
368 if (eglBase != null) {
369 eglBase.release();
370 this.eglBase = null;
371 }
372 }
373
374 synchronized void verifyClosed() {
375 if (this.peerConnection != null
376 || this.eglBase != null
377 || this.localVideoTrack != null
378 || this.remoteVideoTrack != null) {
379 final IllegalStateException e =
380 new IllegalStateException("WebRTCWrapper hasn't been closed properly");
381 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
382 throw e;
383 }
384 }
385
386 boolean isCameraSwitchable() {
387 final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
388 return videoSourceWrapper != null && videoSourceWrapper.isCameraSwitchable();
389 }
390
391 boolean isFrontCamera() {
392 final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
393 return videoSourceWrapper == null || videoSourceWrapper.isFrontCamera();
394 }
395
396 ListenableFuture<Boolean> switchCamera() {
397 final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
398 if (videoSourceWrapper == null) {
399 return Futures.immediateFailedFuture(
400 new IllegalStateException("VideoSourceWrapper has not been initialized"));
401 }
402 return videoSourceWrapper.switchCamera();
403 }
404
405 boolean isMicrophoneEnabled() {
406 final Optional<AudioTrack> audioTrack = TrackWrapper.get(this.localAudioTrack);
407 if (audioTrack.isPresent()) {
408 try {
409 return audioTrack.get().enabled();
410 } catch (final IllegalStateException e) {
411 // sometimes UI might still be rendering the buttons when a background thread has
412 // already ended the call
413 return false;
414 }
415 } else {
416 throw new IllegalStateException("Local audio track does not exist (yet)");
417 }
418 }
419
420 boolean setMicrophoneEnabled(final boolean enabled) {
421 final Optional<AudioTrack> audioTrack = TrackWrapper.get(this.localAudioTrack);
422 if (audioTrack.isPresent()) {
423 try {
424 audioTrack.get().setEnabled(enabled);
425 return true;
426 } catch (final IllegalStateException e) {
427 Log.d(Config.LOGTAG, "unable to toggle microphone", e);
428 // ignoring race condition in case MediaStreamTrack has been disposed
429 return false;
430 }
431 } else {
432 throw new IllegalStateException("Local audio track does not exist (yet)");
433 }
434 }
435
436 boolean isVideoEnabled() {
437 final Optional<VideoTrack> videoTrack = TrackWrapper.get(this.localVideoTrack);
438 if (videoTrack.isPresent()) {
439 return videoTrack.get().enabled();
440 }
441 return false;
442 }
443
444 void setVideoEnabled(final boolean enabled) {
445 final Optional<VideoTrack> videoTrack = TrackWrapper.get(this.localVideoTrack);
446 if (videoTrack.isPresent()) {
447 videoTrack.get().setEnabled(enabled);
448 return;
449 }
450 throw new IllegalStateException("Local video track does not exist");
451 }
452
453 synchronized ListenableFuture<SessionDescription> setLocalDescription() {
454 return Futures.transformAsync(
455 getPeerConnectionFuture(),
456 peerConnection -> {
457 if (peerConnection == null) {
458 return Futures.immediateFailedFuture(
459 new IllegalStateException("PeerConnection was null"));
460 }
461 final SettableFuture<SessionDescription> future = SettableFuture.create();
462 peerConnection.setLocalDescription(
463 new SetSdpObserver() {
464 @Override
465 public void onSetSuccess() {
466 final SessionDescription description =
467 peerConnection.getLocalDescription();
468 Log.d(EXTENDED_LOGGING_TAG, "set local description:");
469 logDescription(description);
470 future.set(description);
471 }
472
473 @Override
474 public void onSetFailure(final String message) {
475 future.setException(
476 new FailureToSetDescriptionException(message));
477 }
478 });
479 return future;
480 },
481 MoreExecutors.directExecutor());
482 }
483
484 private static void logDescription(final SessionDescription sessionDescription) {
485 for (final String line :
486 sessionDescription.description.split(
487 eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
488 Log.d(EXTENDED_LOGGING_TAG, line);
489 }
490 }
491
492 synchronized ListenableFuture<Void> setRemoteDescription(
493 final SessionDescription sessionDescription) {
494 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
495 logDescription(sessionDescription);
496 return Futures.transformAsync(
497 getPeerConnectionFuture(),
498 peerConnection -> {
499 if (peerConnection == null) {
500 return Futures.immediateFailedFuture(
501 new IllegalStateException("PeerConnection was null"));
502 }
503 final SettableFuture<Void> future = SettableFuture.create();
504 peerConnection.setRemoteDescription(
505 new SetSdpObserver() {
506 @Override
507 public void onSetSuccess() {
508 future.set(null);
509 }
510
511 @Override
512 public void onSetFailure(final String message) {
513 future.setException(
514 new FailureToSetDescriptionException(message));
515 }
516 },
517 sessionDescription);
518 return future;
519 },
520 MoreExecutors.directExecutor());
521 }
522
523 @Nonnull
524 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
525 final PeerConnection peerConnection = this.peerConnection;
526 if (peerConnection == null) {
527 return Futures.immediateFailedFuture(new PeerConnectionNotInitialized());
528 } else {
529 return Futures.immediateFuture(peerConnection);
530 }
531 }
532
533 private PeerConnection requirePeerConnection() {
534 final PeerConnection peerConnection = this.peerConnection;
535 if (peerConnection == null) {
536 throw new PeerConnectionNotInitialized();
537 }
538 return peerConnection;
539 }
540
541 void addIceCandidate(IceCandidate iceCandidate) {
542 requirePeerConnection().addIceCandidate(iceCandidate);
543 }
544
545 PeerConnection.PeerConnectionState getState() {
546 return requirePeerConnection().connectionState();
547 }
548
549 public PeerConnection.SignalingState getSignalingState() {
550 return requirePeerConnection().signalingState();
551 }
552
553 EglBase.Context getEglBaseContext() {
554 return this.eglBase.getEglBaseContext();
555 }
556
557 Optional<VideoTrack> getLocalVideoTrack() {
558 return TrackWrapper.get(this.localVideoTrack);
559 }
560
561 Optional<VideoTrack> getRemoteVideoTrack() {
562 return Optional.fromNullable(this.remoteVideoTrack);
563 }
564
565 private Context requireContext() {
566 final Context context = this.context;
567 if (context == null) {
568 throw new IllegalStateException("call setup first");
569 }
570 return context;
571 }
572
573 AppRTCAudioManager getAudioManager() {
574 return appRTCAudioManager;
575 }
576
577 void execute(final Runnable command) {
578 executorService.execute(command);
579 }
580
581 public interface EventCallback {
582 void onIceCandidate(IceCandidate iceCandidate);
583
584 void onConnectionChange(PeerConnection.PeerConnectionState newState);
585
586 void onAudioDeviceChanged(
587 AppRTCAudioManager.AudioDevice selectedAudioDevice,
588 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
589
590 void onRenegotiationNeeded();
591 }
592
593 private abstract static class SetSdpObserver implements SdpObserver {
594
595 @Override
596 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
597 throw new IllegalStateException("Not able to use SetSdpObserver");
598 }
599
600 @Override
601 public void onCreateFailure(String s) {
602 throw new IllegalStateException("Not able to use SetSdpObserver");
603 }
604 }
605
606 static class InitializationException extends Exception {
607
608 private InitializationException(final String message, final Throwable throwable) {
609 super(message, throwable);
610 }
611
612 private InitializationException(final String message) {
613 super(message);
614 }
615 }
616
617 public static class PeerConnectionNotInitialized extends IllegalStateException {
618
619 private PeerConnectionNotInitialized() {
620 super("initialize PeerConnection first");
621 }
622 }
623
624 private static class FailureToSetDescriptionException extends IllegalArgumentException {
625 public FailureToSetDescriptionException(String message) {
626 super(message);
627 }
628 }
629
630}