1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Throwables;
12import com.google.common.collect.ImmutableSet;
13import com.google.common.collect.Iterables;
14import com.google.common.util.concurrent.Futures;
15import com.google.common.util.concurrent.ListenableFuture;
16import com.google.common.util.concurrent.MoreExecutors;
17import com.google.common.util.concurrent.SettableFuture;
18
19import org.webrtc.AudioSource;
20import org.webrtc.AudioTrack;
21import org.webrtc.Camera1Enumerator;
22import org.webrtc.Camera2Enumerator;
23import org.webrtc.CameraEnumerationAndroid;
24import org.webrtc.CameraEnumerator;
25import org.webrtc.CameraVideoCapturer;
26import org.webrtc.CandidatePairChangeEvent;
27import org.webrtc.DataChannel;
28import org.webrtc.DefaultVideoDecoderFactory;
29import org.webrtc.DefaultVideoEncoderFactory;
30import org.webrtc.EglBase;
31import org.webrtc.IceCandidate;
32import org.webrtc.MediaConstraints;
33import org.webrtc.MediaStream;
34import org.webrtc.MediaStreamTrack;
35import org.webrtc.PeerConnection;
36import org.webrtc.PeerConnectionFactory;
37import org.webrtc.RtpReceiver;
38import org.webrtc.RtpTransceiver;
39import org.webrtc.SdpObserver;
40import org.webrtc.SessionDescription;
41import org.webrtc.SurfaceTextureHelper;
42import org.webrtc.VideoSource;
43import org.webrtc.VideoTrack;
44
45import java.util.ArrayList;
46import java.util.Collections;
47import java.util.List;
48import java.util.Set;
49
50import javax.annotation.Nonnull;
51import javax.annotation.Nullable;
52
53import eu.siacs.conversations.Config;
54import eu.siacs.conversations.services.AppRTCAudioManager;
55
56public class WebRTCWrapper {
57
58 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
59
60 private static final int CAPTURING_RESOLUTION = 1920;
61 private static final int CAPTURING_MAX_FRAME_RATE = 30;
62
63 private final EventCallback eventCallback;
64 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
65 @Override
66 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
67 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
68 }
69 };
70 private final Handler mainHandler = new Handler(Looper.getMainLooper());
71 private VideoTrack localVideoTrack = null;
72 private VideoTrack remoteVideoTrack = null;
73 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
74 @Override
75 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
76 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
77 //this is called after removeTrack or addTrack
78 //and should then trigger a content-add or content-remove or something
79 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
80 }
81
82 @Override
83 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
84 eventCallback.onConnectionChange(newState);
85 }
86
87 @Override
88 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
89
90 }
91
92 @Override
93 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
94 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
95 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
96 }
97
98 @Override
99 public void onIceConnectionReceivingChange(boolean b) {
100
101 }
102
103 @Override
104 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
105 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
106 }
107
108 @Override
109 public void onIceCandidate(IceCandidate iceCandidate) {
110 eventCallback.onIceCandidate(iceCandidate);
111 }
112
113 @Override
114 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
115
116 }
117
118 @Override
119 public void onAddStream(MediaStream mediaStream) {
120 Log.d(EXTENDED_LOGGING_TAG, "onAddStream(numAudioTracks=" + mediaStream.audioTracks.size() + ",numVideoTracks=" + mediaStream.videoTracks.size() + ")");
121 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
122 if (videoTracks.size() > 0) {
123 remoteVideoTrack = videoTracks.get(0);
124 Log.d(Config.LOGTAG, "remote video track enabled?=" + remoteVideoTrack.enabled());
125 } else {
126 Log.d(Config.LOGTAG, "no remote video tracks found");
127 }
128 }
129
130 @Override
131 public void onRemoveStream(MediaStream mediaStream) {
132
133 }
134
135 @Override
136 public void onDataChannel(DataChannel dataChannel) {
137
138 }
139
140 @Override
141 public void onRenegotiationNeeded() {
142
143 }
144
145 @Override
146 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
147 final MediaStreamTrack track = rtpReceiver.track();
148 Log.d(EXTENDED_LOGGING_TAG, "onAddTrack(kind=" + (track == null ? "null" : track.kind()) + ",numMediaStreams=" + mediaStreams.length + ")");
149 }
150
151 @Override
152 public void onTrack(RtpTransceiver transceiver) {
153 Log.d(EXTENDED_LOGGING_TAG, "onTrack(mid=" + transceiver.getMid() + ",media=" + transceiver.getMediaType() + ")");
154 }
155 };
156 @Nullable
157 private PeerConnection peerConnection = null;
158 private AudioTrack localAudioTrack = null;
159 private AppRTCAudioManager appRTCAudioManager = null;
160 private Context context = null;
161 private EglBase eglBase = null;
162 private CapturerChoice capturerChoice;
163
164 WebRTCWrapper(final EventCallback eventCallback) {
165 this.eventCallback = eventCallback;
166 }
167
168 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) throws InitializationException {
169 try {
170 PeerConnectionFactory.initialize(
171 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
172 );
173 } catch (final UnsatisfiedLinkError e) {
174 throw new InitializationException(e);
175 }
176 this.eglBase = EglBase.create();
177 this.context = context;
178 mainHandler.post(() -> {
179 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
180 appRTCAudioManager.start(audioManagerEvents);
181 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
182 });
183 }
184
185 synchronized void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
186 Preconditions.checkState(this.eglBase != null);
187 Preconditions.checkNotNull(media);
188 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
189 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
190 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
191 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
192 .createPeerConnectionFactory();
193
194
195 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
196
197 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
198
199 if (optionalCapturerChoice.isPresent()) {
200 this.capturerChoice = optionalCapturerChoice.get();
201 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
202 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
203 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
204 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
205 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
206 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
207
208 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
209
210 stream.addTrack(this.localVideoTrack);
211 }
212
213
214 if (media.contains(Media.AUDIO)) {
215 //set up audio track
216 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
217 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
218 stream.addTrack(this.localAudioTrack);
219 }
220
221
222 final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
223 rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp
224 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
225 if (peerConnection == null) {
226 throw new InitializationException("Unable to create PeerConnection");
227 }
228 peerConnection.addStream(stream);
229 peerConnection.setAudioPlayout(true);
230 peerConnection.setAudioRecording(true);
231 this.peerConnection = peerConnection;
232 }
233
234 synchronized void close() {
235 final PeerConnection peerConnection = this.peerConnection;
236 final CapturerChoice capturerChoice = this.capturerChoice;
237 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
238 final EglBase eglBase = this.eglBase;
239 if (peerConnection != null) {
240 dispose(peerConnection);
241 this.peerConnection = null;
242 }
243 if (audioManager != null) {
244 mainHandler.post(audioManager::stop);
245 }
246 this.localVideoTrack = null;
247 this.remoteVideoTrack = null;
248 if (capturerChoice != null) {
249 try {
250 capturerChoice.cameraVideoCapturer.stopCapture();
251 } catch (InterruptedException e) {
252 Log.e(Config.LOGTAG, "unable to stop capturing");
253 }
254 }
255 if (eglBase != null) {
256 eglBase.release();
257 this.eglBase = null;
258 }
259 }
260
261 private static void dispose(final PeerConnection peerConnection) {
262 try {
263 peerConnection.dispose();
264 } catch (final IllegalStateException e) {
265 Log.e(Config.LOGTAG, "unable to dispose of peer connection", e);
266 }
267 }
268
269 synchronized void verifyClosed() {
270 if (this.peerConnection != null
271 || this.eglBase != null
272 || this.localVideoTrack != null
273 || this.remoteVideoTrack != null) {
274 final IllegalStateException e = new IllegalStateException("WebRTCWrapper hasn't been closed properly");
275 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
276 throw e;
277 }
278 }
279
280 boolean isCameraSwitchable() {
281 final CapturerChoice capturerChoice = this.capturerChoice;
282 return capturerChoice != null && capturerChoice.availableCameras.size() > 1;
283 }
284
285 boolean isFrontCamera() {
286 final CapturerChoice capturerChoice = this.capturerChoice;
287 return capturerChoice == null || capturerChoice.isFrontCamera;
288 }
289
290 ListenableFuture<Boolean> switchCamera() {
291 final CapturerChoice capturerChoice = this.capturerChoice;
292 if (capturerChoice == null) {
293 return Futures.immediateFailedFuture(new IllegalStateException("CameraCapturer has not been initialized"));
294 }
295 final SettableFuture<Boolean> future = SettableFuture.create();
296 capturerChoice.cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() {
297 @Override
298 public void onCameraSwitchDone(boolean isFrontCamera) {
299 capturerChoice.isFrontCamera = isFrontCamera;
300 future.set(isFrontCamera);
301 }
302
303 @Override
304 public void onCameraSwitchError(final String message) {
305 future.setException(new IllegalStateException(String.format("Unable to switch camera %s", message)));
306 }
307 });
308 return future;
309 }
310
311 boolean isMicrophoneEnabled() {
312 final AudioTrack audioTrack = this.localAudioTrack;
313 if (audioTrack == null) {
314 throw new IllegalStateException("Local audio track does not exist (yet)");
315 }
316 try {
317 return audioTrack.enabled();
318 } catch (final IllegalStateException e) {
319 //sometimes UI might still be rendering the buttons when a background thread has already ended the call
320 return false;
321 }
322 }
323
324 void setMicrophoneEnabled(final boolean enabled) {
325 final AudioTrack audioTrack = this.localAudioTrack;
326 if (audioTrack == null) {
327 throw new IllegalStateException("Local audio track does not exist (yet)");
328 }
329 audioTrack.setEnabled(enabled);
330 }
331
332 boolean isVideoEnabled() {
333 final VideoTrack videoTrack = this.localVideoTrack;
334 if (videoTrack == null) {
335 throw new IllegalStateException("Local video track does not exist");
336 }
337 return videoTrack.enabled();
338 }
339
340 void setVideoEnabled(final boolean enabled) {
341 final VideoTrack videoTrack = this.localVideoTrack;
342 if (videoTrack == null) {
343 throw new IllegalStateException("Local video track does not exist");
344 }
345 videoTrack.setEnabled(enabled);
346 }
347
348 ListenableFuture<SessionDescription> createOffer() {
349 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
350 final SettableFuture<SessionDescription> future = SettableFuture.create();
351 peerConnection.createOffer(new CreateSdpObserver() {
352 @Override
353 public void onCreateSuccess(SessionDescription sessionDescription) {
354 future.set(sessionDescription);
355 }
356
357 @Override
358 public void onCreateFailure(String s) {
359 future.setException(new IllegalStateException("Unable to create offer: " + s));
360 }
361 }, new MediaConstraints());
362 return future;
363 }, MoreExecutors.directExecutor());
364 }
365
366 ListenableFuture<SessionDescription> createAnswer() {
367 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
368 final SettableFuture<SessionDescription> future = SettableFuture.create();
369 peerConnection.createAnswer(new CreateSdpObserver() {
370 @Override
371 public void onCreateSuccess(SessionDescription sessionDescription) {
372 future.set(sessionDescription);
373 }
374
375 @Override
376 public void onCreateFailure(String s) {
377 future.setException(new IllegalStateException("Unable to create answer: " + s));
378 }
379 }, new MediaConstraints());
380 return future;
381 }, MoreExecutors.directExecutor());
382 }
383
384 ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
385 Log.d(EXTENDED_LOGGING_TAG, "setting local description:");
386 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
387 Log.d(EXTENDED_LOGGING_TAG, line);
388 }
389 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
390 final SettableFuture<Void> future = SettableFuture.create();
391 peerConnection.setLocalDescription(new SetSdpObserver() {
392 @Override
393 public void onSetSuccess() {
394 future.set(null);
395 }
396
397 @Override
398 public void onSetFailure(final String s) {
399 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
400
401 }
402 }, sessionDescription);
403 return future;
404 }, MoreExecutors.directExecutor());
405 }
406
407 ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
408 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
409 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
410 Log.d(EXTENDED_LOGGING_TAG, line);
411 }
412 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
413 final SettableFuture<Void> future = SettableFuture.create();
414 peerConnection.setRemoteDescription(new SetSdpObserver() {
415 @Override
416 public void onSetSuccess() {
417 future.set(null);
418 }
419
420 @Override
421 public void onSetFailure(String s) {
422 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
423
424 }
425 }, sessionDescription);
426 return future;
427 }, MoreExecutors.directExecutor());
428 }
429
430 @Nonnull
431 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
432 final PeerConnection peerConnection = this.peerConnection;
433 if (peerConnection == null) {
434 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
435 } else {
436 return Futures.immediateFuture(peerConnection);
437 }
438 }
439
440 void addIceCandidate(IceCandidate iceCandidate) {
441 requirePeerConnection().addIceCandidate(iceCandidate);
442 }
443
444 private CameraEnumerator getCameraEnumerator() {
445 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
446 return new Camera2Enumerator(requireContext());
447 } else {
448 return new Camera1Enumerator();
449 }
450 }
451
452 private Optional<CapturerChoice> getVideoCapturer() {
453 final CameraEnumerator enumerator = getCameraEnumerator();
454 final Set<String> deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames());
455 for (final String deviceName : deviceNames) {
456 if (enumerator.isFrontFacing(deviceName)) {
457 final CapturerChoice capturerChoice = of(enumerator, deviceName, deviceNames);
458 if (capturerChoice == null) {
459 return Optional.absent();
460 }
461 capturerChoice.isFrontCamera = true;
462 return Optional.of(capturerChoice);
463 }
464 }
465 if (deviceNames.size() == 0) {
466 return Optional.absent();
467 } else {
468 return Optional.fromNullable(of(enumerator, Iterables.get(deviceNames, 0), deviceNames));
469 }
470 }
471
472 @Nullable
473 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName, Set<String> availableCameras) {
474 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
475 if (capturer == null) {
476 return null;
477 }
478 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
479 Collections.sort(choices, (a, b) -> b.width - a.width);
480 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
481 if (captureFormat.width <= CAPTURING_RESOLUTION) {
482 return new CapturerChoice(capturer, captureFormat, availableCameras);
483 }
484 }
485 return null;
486 }
487
488 public PeerConnection.PeerConnectionState getState() {
489 return requirePeerConnection().connectionState();
490 }
491
492 EglBase.Context getEglBaseContext() {
493 return this.eglBase.getEglBaseContext();
494 }
495
496 Optional<VideoTrack> getLocalVideoTrack() {
497 return Optional.fromNullable(this.localVideoTrack);
498 }
499
500 Optional<VideoTrack> getRemoteVideoTrack() {
501 return Optional.fromNullable(this.remoteVideoTrack);
502 }
503
504 private PeerConnection requirePeerConnection() {
505 final PeerConnection peerConnection = this.peerConnection;
506 if (peerConnection == null) {
507 throw new IllegalStateException("initialize PeerConnection first");
508 }
509 return peerConnection;
510 }
511
512 private Context requireContext() {
513 final Context context = this.context;
514 if (context == null) {
515 throw new IllegalStateException("call setup first");
516 }
517 return context;
518 }
519
520 AppRTCAudioManager getAudioManager() {
521 return appRTCAudioManager;
522 }
523
524 public interface EventCallback {
525 void onIceCandidate(IceCandidate iceCandidate);
526
527 void onConnectionChange(PeerConnection.PeerConnectionState newState);
528
529 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
530 }
531
532 private static abstract class SetSdpObserver implements SdpObserver {
533
534 @Override
535 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
536 throw new IllegalStateException("Not able to use SetSdpObserver");
537 }
538
539 @Override
540 public void onCreateFailure(String s) {
541 throw new IllegalStateException("Not able to use SetSdpObserver");
542 }
543
544 }
545
546 private static abstract class CreateSdpObserver implements SdpObserver {
547
548
549 @Override
550 public void onSetSuccess() {
551 throw new IllegalStateException("Not able to use CreateSdpObserver");
552 }
553
554
555 @Override
556 public void onSetFailure(String s) {
557 throw new IllegalStateException("Not able to use CreateSdpObserver");
558 }
559 }
560
561 static class InitializationException extends Exception {
562
563 private InitializationException(final Throwable throwable) {
564 super(throwable);
565 }
566
567 private InitializationException(final String message) {
568 super(message);
569 }
570 }
571
572 private static class CapturerChoice {
573 private final CameraVideoCapturer cameraVideoCapturer;
574 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
575 private final Set<String> availableCameras;
576 private boolean isFrontCamera = false;
577
578 CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat, Set<String> cameras) {
579 this.cameraVideoCapturer = cameraVideoCapturer;
580 this.captureFormat = captureFormat;
581 this.availableCameras = cameras;
582 }
583
584 int getFrameRate() {
585 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
586 }
587 }
588}