1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.collect.ImmutableSet;
12import com.google.common.collect.Iterables;
13import com.google.common.util.concurrent.Futures;
14import com.google.common.util.concurrent.ListenableFuture;
15import com.google.common.util.concurrent.MoreExecutors;
16import com.google.common.util.concurrent.SettableFuture;
17
18import org.webrtc.AudioSource;
19import org.webrtc.AudioTrack;
20import org.webrtc.Camera1Enumerator;
21import org.webrtc.Camera2Enumerator;
22import org.webrtc.CameraEnumerationAndroid;
23import org.webrtc.CameraEnumerator;
24import org.webrtc.CameraVideoCapturer;
25import org.webrtc.CandidatePairChangeEvent;
26import org.webrtc.DataChannel;
27import org.webrtc.DefaultVideoDecoderFactory;
28import org.webrtc.DefaultVideoEncoderFactory;
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.MediaConstraints;
32import org.webrtc.MediaStream;
33import org.webrtc.MediaStreamTrack;
34import org.webrtc.PeerConnection;
35import org.webrtc.PeerConnectionFactory;
36import org.webrtc.RtpReceiver;
37import org.webrtc.RtpTransceiver;
38import org.webrtc.SdpObserver;
39import org.webrtc.SessionDescription;
40import org.webrtc.SurfaceTextureHelper;
41import org.webrtc.VideoSource;
42import org.webrtc.VideoTrack;
43
44import java.util.ArrayList;
45import java.util.Collections;
46import java.util.List;
47import java.util.Set;
48
49import javax.annotation.Nonnull;
50import javax.annotation.Nullable;
51
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.services.AppRTCAudioManager;
54
55public class WebRTCWrapper {
56
57 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
58
59 private static final int CAPTURING_RESOLUTION = 1920;
60 private static final int CAPTURING_MAX_FRAME_RATE = 30;
61
62 private final EventCallback eventCallback;
63 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
64 @Override
65 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
66 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
67 }
68 };
69 private final Handler mainHandler = new Handler(Looper.getMainLooper());
70 private VideoTrack localVideoTrack = null;
71 private VideoTrack remoteVideoTrack = null;
72 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
73 @Override
74 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
75 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
76 //this is called after removeTrack or addTrack
77 //and should then trigger a content-add or content-remove or something
78 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
79 }
80
81 @Override
82 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
83 eventCallback.onConnectionChange(newState);
84 }
85
86 @Override
87 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
88
89 }
90
91 @Override
92 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
93 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
94 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
95 }
96
97 @Override
98 public void onIceConnectionReceivingChange(boolean b) {
99
100 }
101
102 @Override
103 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
104 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
105 }
106
107 @Override
108 public void onIceCandidate(IceCandidate iceCandidate) {
109 eventCallback.onIceCandidate(iceCandidate);
110 }
111
112 @Override
113 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
114
115 }
116
117 @Override
118 public void onAddStream(MediaStream mediaStream) {
119 Log.d(EXTENDED_LOGGING_TAG, "onAddStream(numAudioTracks=" + mediaStream.audioTracks.size() + ",numVideoTracks=" + mediaStream.videoTracks.size() + ")");
120 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
121 if (videoTracks.size() > 0) {
122 remoteVideoTrack = videoTracks.get(0);
123 Log.d(Config.LOGTAG, "remote video track enabled?=" + remoteVideoTrack.enabled());
124 } else {
125 Log.d(Config.LOGTAG, "no remote video tracks found");
126 }
127 }
128
129 @Override
130 public void onRemoveStream(MediaStream mediaStream) {
131
132 }
133
134 @Override
135 public void onDataChannel(DataChannel dataChannel) {
136
137 }
138
139 @Override
140 public void onRenegotiationNeeded() {
141
142 }
143
144 @Override
145 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
146 final MediaStreamTrack track = rtpReceiver.track();
147 Log.d(EXTENDED_LOGGING_TAG, "onAddTrack(kind=" + (track == null ? "null" : track.kind()) + ",numMediaStreams=" + mediaStreams.length + ")");
148 }
149
150 @Override
151 public void onTrack(RtpTransceiver transceiver) {
152 Log.d(EXTENDED_LOGGING_TAG, "onTrack(mid=" + transceiver.getMid() + ",media=" + transceiver.getMediaType() + ")");
153 }
154 };
155 @Nullable
156 private PeerConnection peerConnection = null;
157 private AudioTrack localAudioTrack = null;
158 private AppRTCAudioManager appRTCAudioManager = null;
159 private Context context = null;
160 private EglBase eglBase = null;
161 private CapturerChoice capturerChoice;
162
163 WebRTCWrapper(final EventCallback eventCallback) {
164 this.eventCallback = eventCallback;
165 }
166
167 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
168 PeerConnectionFactory.initialize(
169 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
170 );
171 this.eglBase = EglBase.create();
172 this.context = context;
173 mainHandler.post(() -> {
174 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
175 appRTCAudioManager.start(audioManagerEvents);
176 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
177 });
178 }
179
180 synchronized void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
181 Preconditions.checkState(this.eglBase != null);
182 Preconditions.checkNotNull(media);
183 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
184 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
185 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
186 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
187 .createPeerConnectionFactory();
188
189
190 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
191
192 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
193
194 if (optionalCapturerChoice.isPresent()) {
195 this.capturerChoice = optionalCapturerChoice.get();
196 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
197 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
198 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
199 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
200 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
201 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
202
203 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
204
205 stream.addTrack(this.localVideoTrack);
206 }
207
208
209 if (media.contains(Media.AUDIO)) {
210 //set up audio track
211 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
212 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
213 stream.addTrack(this.localAudioTrack);
214 }
215
216
217 final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
218 rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp
219 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
220 if (peerConnection == null) {
221 throw new InitializationException("Unable to create PeerConnection");
222 }
223 peerConnection.addStream(stream);
224 peerConnection.setAudioPlayout(true);
225 peerConnection.setAudioRecording(true);
226 this.peerConnection = peerConnection;
227 }
228
229 synchronized void close() {
230 final PeerConnection peerConnection = this.peerConnection;
231 final CapturerChoice capturerChoice = this.capturerChoice;
232 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
233 final EglBase eglBase = this.eglBase;
234 if (peerConnection != null) {
235 dispose(peerConnection);
236 this.peerConnection = null;
237 }
238 if (audioManager != null) {
239 mainHandler.post(audioManager::stop);
240 }
241 this.localVideoTrack = null;
242 this.remoteVideoTrack = null;
243 if (capturerChoice != null) {
244 try {
245 capturerChoice.cameraVideoCapturer.stopCapture();
246 } catch (InterruptedException e) {
247 Log.e(Config.LOGTAG, "unable to stop capturing");
248 }
249 }
250 if (eglBase != null) {
251 eglBase.release();
252 this.eglBase = null;
253 }
254 }
255
256 private static void dispose(final PeerConnection peerConnection) {
257 try {
258 peerConnection.dispose();
259 } catch (final IllegalStateException e) {
260 Log.e(Config.LOGTAG, "unable to dispose of peer connection", e);
261 }
262 }
263
264 synchronized void verifyClosed() {
265 if (this.peerConnection != null
266 || this.eglBase != null
267 || this.localVideoTrack != null
268 || this.remoteVideoTrack != null) {
269 final IllegalStateException e = new IllegalStateException("WebRTCWrapper hasn't been closed properly");
270 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
271 throw e;
272 }
273 }
274
275 boolean isCameraSwitchable() {
276 final CapturerChoice capturerChoice = this.capturerChoice;
277 return capturerChoice != null && capturerChoice.availableCameras.size() > 1;
278 }
279
280 boolean isFrontCamera() {
281 final CapturerChoice capturerChoice = this.capturerChoice;
282 return capturerChoice == null || capturerChoice.isFrontCamera;
283 }
284
285 ListenableFuture<Boolean> switchCamera() {
286 final CapturerChoice capturerChoice = this.capturerChoice;
287 if (capturerChoice == null) {
288 return Futures.immediateFailedFuture(new IllegalStateException("CameraCapturer has not been initialized"));
289 }
290 final SettableFuture<Boolean> future = SettableFuture.create();
291 capturerChoice.cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() {
292 @Override
293 public void onCameraSwitchDone(boolean isFrontCamera) {
294 capturerChoice.isFrontCamera = isFrontCamera;
295 future.set(isFrontCamera);
296 }
297
298 @Override
299 public void onCameraSwitchError(final String message) {
300 future.setException(new IllegalStateException(String.format("Unable to switch camera %s", message)));
301 }
302 });
303 return future;
304 }
305
306 boolean isMicrophoneEnabled() {
307 final AudioTrack audioTrack = this.localAudioTrack;
308 if (audioTrack == null) {
309 throw new IllegalStateException("Local audio track does not exist (yet)");
310 }
311 try {
312 return audioTrack.enabled();
313 } catch (final IllegalStateException e) {
314 //sometimes UI might still be rendering the buttons when a background thread has already ended the call
315 return false;
316 }
317 }
318
319 void setMicrophoneEnabled(final boolean enabled) {
320 final AudioTrack audioTrack = this.localAudioTrack;
321 if (audioTrack == null) {
322 throw new IllegalStateException("Local audio track does not exist (yet)");
323 }
324 audioTrack.setEnabled(enabled);
325 }
326
327 boolean isVideoEnabled() {
328 final VideoTrack videoTrack = this.localVideoTrack;
329 if (videoTrack == null) {
330 throw new IllegalStateException("Local video track does not exist");
331 }
332 return videoTrack.enabled();
333 }
334
335 void setVideoEnabled(final boolean enabled) {
336 final VideoTrack videoTrack = this.localVideoTrack;
337 if (videoTrack == null) {
338 throw new IllegalStateException("Local video track does not exist");
339 }
340 videoTrack.setEnabled(enabled);
341 }
342
343 ListenableFuture<SessionDescription> createOffer() {
344 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
345 final SettableFuture<SessionDescription> future = SettableFuture.create();
346 peerConnection.createOffer(new CreateSdpObserver() {
347 @Override
348 public void onCreateSuccess(SessionDescription sessionDescription) {
349 future.set(sessionDescription);
350 }
351
352 @Override
353 public void onCreateFailure(String s) {
354 future.setException(new IllegalStateException("Unable to create offer: " + s));
355 }
356 }, new MediaConstraints());
357 return future;
358 }, MoreExecutors.directExecutor());
359 }
360
361 ListenableFuture<SessionDescription> createAnswer() {
362 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
363 final SettableFuture<SessionDescription> future = SettableFuture.create();
364 peerConnection.createAnswer(new CreateSdpObserver() {
365 @Override
366 public void onCreateSuccess(SessionDescription sessionDescription) {
367 future.set(sessionDescription);
368 }
369
370 @Override
371 public void onCreateFailure(String s) {
372 future.setException(new IllegalStateException("Unable to create answer: " + s));
373 }
374 }, new MediaConstraints());
375 return future;
376 }, MoreExecutors.directExecutor());
377 }
378
379 ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
380 Log.d(EXTENDED_LOGGING_TAG, "setting local description:");
381 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
382 Log.d(EXTENDED_LOGGING_TAG, line);
383 }
384 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
385 final SettableFuture<Void> future = SettableFuture.create();
386 peerConnection.setLocalDescription(new SetSdpObserver() {
387 @Override
388 public void onSetSuccess() {
389 future.set(null);
390 }
391
392 @Override
393 public void onSetFailure(final String s) {
394 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
395
396 }
397 }, sessionDescription);
398 return future;
399 }, MoreExecutors.directExecutor());
400 }
401
402 ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
403 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
404 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
405 Log.d(EXTENDED_LOGGING_TAG, line);
406 }
407 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
408 final SettableFuture<Void> future = SettableFuture.create();
409 peerConnection.setRemoteDescription(new SetSdpObserver() {
410 @Override
411 public void onSetSuccess() {
412 future.set(null);
413 }
414
415 @Override
416 public void onSetFailure(String s) {
417 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
418
419 }
420 }, sessionDescription);
421 return future;
422 }, MoreExecutors.directExecutor());
423 }
424
425 @Nonnull
426 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
427 final PeerConnection peerConnection = this.peerConnection;
428 if (peerConnection == null) {
429 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
430 } else {
431 return Futures.immediateFuture(peerConnection);
432 }
433 }
434
435 void addIceCandidate(IceCandidate iceCandidate) {
436 requirePeerConnection().addIceCandidate(iceCandidate);
437 }
438
439 private CameraEnumerator getCameraEnumerator() {
440 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
441 return new Camera2Enumerator(requireContext());
442 } else {
443 return new Camera1Enumerator();
444 }
445 }
446
447 private Optional<CapturerChoice> getVideoCapturer() {
448 final CameraEnumerator enumerator = getCameraEnumerator();
449 final Set<String> deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames());
450 for (final String deviceName : deviceNames) {
451 if (enumerator.isFrontFacing(deviceName)) {
452 final CapturerChoice capturerChoice = of(enumerator, deviceName, deviceNames);
453 if (capturerChoice == null) {
454 return Optional.absent();
455 }
456 capturerChoice.isFrontCamera = true;
457 return Optional.of(capturerChoice);
458 }
459 }
460 if (deviceNames.size() == 0) {
461 return Optional.absent();
462 } else {
463 return Optional.fromNullable(of(enumerator, Iterables.get(deviceNames, 0), deviceNames));
464 }
465 }
466
467 @Nullable
468 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName, Set<String> availableCameras) {
469 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
470 if (capturer == null) {
471 return null;
472 }
473 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
474 Collections.sort(choices, (a, b) -> b.width - a.width);
475 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
476 if (captureFormat.width <= CAPTURING_RESOLUTION) {
477 return new CapturerChoice(capturer, captureFormat, availableCameras);
478 }
479 }
480 return null;
481 }
482
483 public PeerConnection.PeerConnectionState getState() {
484 return requirePeerConnection().connectionState();
485 }
486
487 EglBase.Context getEglBaseContext() {
488 return this.eglBase.getEglBaseContext();
489 }
490
491 Optional<VideoTrack> getLocalVideoTrack() {
492 return Optional.fromNullable(this.localVideoTrack);
493 }
494
495 Optional<VideoTrack> getRemoteVideoTrack() {
496 return Optional.fromNullable(this.remoteVideoTrack);
497 }
498
499 private PeerConnection requirePeerConnection() {
500 final PeerConnection peerConnection = this.peerConnection;
501 if (peerConnection == null) {
502 throw new IllegalStateException("initialize PeerConnection first");
503 }
504 return peerConnection;
505 }
506
507 private Context requireContext() {
508 final Context context = this.context;
509 if (context == null) {
510 throw new IllegalStateException("call setup first");
511 }
512 return context;
513 }
514
515 AppRTCAudioManager getAudioManager() {
516 return appRTCAudioManager;
517 }
518
519 public interface EventCallback {
520 void onIceCandidate(IceCandidate iceCandidate);
521
522 void onConnectionChange(PeerConnection.PeerConnectionState newState);
523
524 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
525 }
526
527 private static abstract class SetSdpObserver implements SdpObserver {
528
529 @Override
530 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
531 throw new IllegalStateException("Not able to use SetSdpObserver");
532 }
533
534 @Override
535 public void onCreateFailure(String s) {
536 throw new IllegalStateException("Not able to use SetSdpObserver");
537 }
538
539 }
540
541 private static abstract class CreateSdpObserver implements SdpObserver {
542
543
544 @Override
545 public void onSetSuccess() {
546 throw new IllegalStateException("Not able to use CreateSdpObserver");
547 }
548
549
550 @Override
551 public void onSetFailure(String s) {
552 throw new IllegalStateException("Not able to use CreateSdpObserver");
553 }
554 }
555
556 static class InitializationException extends Exception {
557
558 private InitializationException(String message) {
559 super(message);
560 }
561 }
562
563 private static class CapturerChoice {
564 private final CameraVideoCapturer cameraVideoCapturer;
565 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
566 private final Set<String> availableCameras;
567 private boolean isFrontCamera = false;
568
569 CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat, Set<String> cameras) {
570 this.cameraVideoCapturer = cameraVideoCapturer;
571 this.captureFormat = captureFormat;
572 this.availableCameras = cameras;
573 }
574
575 int getFrameRate() {
576 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
577 }
578 }
579}