1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerationAndroid;
21import org.webrtc.CameraEnumerator;
22import org.webrtc.CameraVideoCapturer;
23import org.webrtc.CandidatePairChangeEvent;
24import org.webrtc.DataChannel;
25import org.webrtc.DefaultVideoDecoderFactory;
26import org.webrtc.DefaultVideoEncoderFactory;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.MediaConstraints;
30import org.webrtc.MediaStream;
31import org.webrtc.PeerConnection;
32import org.webrtc.PeerConnectionFactory;
33import org.webrtc.RtpReceiver;
34import org.webrtc.SdpObserver;
35import org.webrtc.SessionDescription;
36import org.webrtc.SurfaceTextureHelper;
37import org.webrtc.VideoSource;
38import org.webrtc.VideoTrack;
39
40import java.util.ArrayList;
41import java.util.Collections;
42import java.util.List;
43import java.util.Set;
44
45import javax.annotation.Nonnull;
46import javax.annotation.Nullable;
47
48import eu.siacs.conversations.Config;
49import eu.siacs.conversations.services.AppRTCAudioManager;
50
51public class WebRTCWrapper {
52
53 private static final int CAPTURING_RESOLUTION = 1920;
54 private static final int CAPTURING_MAX_FRAME_RATE = 30;
55
56 private final EventCallback eventCallback;
57 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
58 @Override
59 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
60 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
61 }
62 };
63 private final Handler mainHandler = new Handler(Looper.getMainLooper());
64 private VideoTrack localVideoTrack = null;
65 private VideoTrack remoteVideoTrack = null;
66 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
67 @Override
68 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
69 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
70 //this is called after removeTrack or addTrack
71 //and should then trigger a content-add or content-remove or something
72 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
73 }
74
75 @Override
76 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
77 eventCallback.onConnectionChange(newState);
78 }
79
80 @Override
81 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
82
83 }
84
85 @Override
86 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
87 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
88 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
89 }
90
91 @Override
92 public void onIceConnectionReceivingChange(boolean b) {
93
94 }
95
96 @Override
97 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
98
99 }
100
101 @Override
102 public void onIceCandidate(IceCandidate iceCandidate) {
103 eventCallback.onIceCandidate(iceCandidate);
104 }
105
106 @Override
107 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
108
109 }
110
111 @Override
112 public void onAddStream(MediaStream mediaStream) {
113 Log.d(Config.LOGTAG, "onAddStream");
114 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
115 if (videoTracks.size() > 0) {
116 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
117 remoteVideoTrack = videoTracks.get(0);
118 }
119 }
120
121 @Override
122 public void onRemoveStream(MediaStream mediaStream) {
123
124 }
125
126 @Override
127 public void onDataChannel(DataChannel dataChannel) {
128
129 }
130
131 @Override
132 public void onRenegotiationNeeded() {
133
134 }
135
136 @Override
137 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
138 Log.d(Config.LOGTAG, "onAddTrack()");
139
140 }
141 };
142 @Nullable
143 private PeerConnection peerConnection = null;
144 private AudioTrack localAudioTrack = null;
145 private AppRTCAudioManager appRTCAudioManager = null;
146 private Context context = null;
147 private EglBase eglBase = null;
148 private CapturerChoice capturerChoice;
149
150 public WebRTCWrapper(final EventCallback eventCallback) {
151 this.eventCallback = eventCallback;
152 }
153
154 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
155 PeerConnectionFactory.initialize(
156 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
157 );
158 this.eglBase = EglBase.create();
159 this.context = context;
160 mainHandler.post(() -> {
161 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
162 appRTCAudioManager.start(audioManagerEvents);
163 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
164 });
165 }
166
167 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
168 Preconditions.checkState(this.eglBase != null);
169 Preconditions.checkNotNull(media);
170 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
171 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
172 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
173 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
174 .createPeerConnectionFactory();
175
176
177 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
178
179 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
180
181 if (optionalCapturerChoice.isPresent()) {
182 this.capturerChoice = optionalCapturerChoice.get();
183 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
184 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
185 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
186 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
187 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
188 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
189
190 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
191
192 stream.addTrack(this.localVideoTrack);
193 }
194
195
196 if (media.contains(Media.AUDIO)) {
197 //set up audio track
198 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
199 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
200 stream.addTrack(this.localAudioTrack);
201 }
202
203
204 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
205 if (peerConnection == null) {
206 throw new InitializationException("Unable to create PeerConnection");
207 }
208 peerConnection.addStream(stream);
209 peerConnection.setAudioPlayout(true);
210 peerConnection.setAudioRecording(true);
211 this.peerConnection = peerConnection;
212 }
213
214 public void close() {
215 final PeerConnection peerConnection = this.peerConnection;
216 final CapturerChoice capturerChoice = this.capturerChoice;
217 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
218 final EglBase eglBase = this.eglBase;
219 if (peerConnection != null) {
220 peerConnection.dispose();
221 }
222 if (audioManager != null) {
223 mainHandler.post(audioManager::stop);
224 }
225 this.localVideoTrack = null;
226 this.remoteVideoTrack = null;
227 if (capturerChoice != null) {
228 try {
229 capturerChoice.cameraVideoCapturer.stopCapture();
230 } catch (InterruptedException e) {
231 Log.e(Config.LOGTAG, "unable to stop capturing");
232 }
233 }
234 if (eglBase != null) {
235 eglBase.release();
236 }
237 }
238
239 boolean isMicrophoneEnabled() {
240 final AudioTrack audioTrack = this.localAudioTrack;
241 if (audioTrack == null) {
242 throw new IllegalStateException("Local audio track does not exist (yet)");
243 }
244 return audioTrack.enabled();
245 }
246
247 void setMicrophoneEnabled(final boolean enabled) {
248 final AudioTrack audioTrack = this.localAudioTrack;
249 if (audioTrack == null) {
250 throw new IllegalStateException("Local audio track does not exist (yet)");
251 }
252 audioTrack.setEnabled(enabled);
253 }
254
255 public boolean isVideoEnabled() {
256 final VideoTrack videoTrack = this.localVideoTrack;
257 if (videoTrack == null) {
258 throw new IllegalStateException("Local video track does not exist");
259 }
260 return videoTrack.enabled();
261 }
262
263 public void setVideoEnabled(final boolean enabled) {
264 final VideoTrack videoTrack = this.localVideoTrack;
265 if (videoTrack == null) {
266 throw new IllegalStateException("Local video track does not exist");
267 }
268 videoTrack.setEnabled(enabled);
269 }
270
271 public ListenableFuture<SessionDescription> createOffer() {
272 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
273 final SettableFuture<SessionDescription> future = SettableFuture.create();
274 peerConnection.createOffer(new CreateSdpObserver() {
275 @Override
276 public void onCreateSuccess(SessionDescription sessionDescription) {
277 future.set(sessionDescription);
278 }
279
280 @Override
281 public void onCreateFailure(String s) {
282 Log.d(Config.LOGTAG, "create failure" + s);
283 future.setException(new IllegalStateException("Unable to create offer: " + s));
284 }
285 }, new MediaConstraints());
286 return future;
287 }, MoreExecutors.directExecutor());
288 }
289
290 public ListenableFuture<SessionDescription> createAnswer() {
291 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
292 final SettableFuture<SessionDescription> future = SettableFuture.create();
293 peerConnection.createAnswer(new CreateSdpObserver() {
294 @Override
295 public void onCreateSuccess(SessionDescription sessionDescription) {
296 future.set(sessionDescription);
297 }
298
299 @Override
300 public void onCreateFailure(String s) {
301 future.setException(new IllegalStateException("Unable to create answer: " + s));
302 }
303 }, new MediaConstraints());
304 return future;
305 }, MoreExecutors.directExecutor());
306 }
307
308 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
309 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
310 final SettableFuture<Void> future = SettableFuture.create();
311 peerConnection.setLocalDescription(new SetSdpObserver() {
312 @Override
313 public void onSetSuccess() {
314 future.set(null);
315 }
316
317 @Override
318 public void onSetFailure(String s) {
319 Log.d(Config.LOGTAG, "unable to set local " + s);
320 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
321
322 }
323 }, sessionDescription);
324 return future;
325 }, MoreExecutors.directExecutor());
326 }
327
328 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
329 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
330 final SettableFuture<Void> future = SettableFuture.create();
331 peerConnection.setRemoteDescription(new SetSdpObserver() {
332 @Override
333 public void onSetSuccess() {
334 future.set(null);
335 }
336
337 @Override
338 public void onSetFailure(String s) {
339 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
340
341 }
342 }, sessionDescription);
343 return future;
344 }, MoreExecutors.directExecutor());
345 }
346
347 @Nonnull
348 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
349 final PeerConnection peerConnection = this.peerConnection;
350 if (peerConnection == null) {
351 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
352 } else {
353 return Futures.immediateFuture(peerConnection);
354 }
355 }
356
357 public void addIceCandidate(IceCandidate iceCandidate) {
358 requirePeerConnection().addIceCandidate(iceCandidate);
359 }
360
361 private CameraEnumerator getCameraEnumerator() {
362 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
363 return new Camera2Enumerator(requireContext());
364 } else {
365 return new Camera1Enumerator();
366 }
367 }
368
369 private Optional<CapturerChoice> getVideoCapturer() {
370 final CameraEnumerator enumerator = getCameraEnumerator();
371 final String[] deviceNames = enumerator.getDeviceNames();
372 for (final String deviceName : deviceNames) {
373 if (enumerator.isFrontFacing(deviceName)) {
374 return Optional.fromNullable(of(enumerator, deviceName));
375 }
376 }
377 if (deviceNames.length == 0) {
378 return Optional.absent();
379 } else {
380 return Optional.fromNullable(of(enumerator, deviceNames[0]));
381 }
382 }
383
384 @Nullable
385 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName) {
386 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
387 if (capturer == null) {
388 return null;
389 }
390 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
391 Collections.sort(choices, (a, b) -> b.width - a.width);
392 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
393 if (captureFormat.width <= CAPTURING_RESOLUTION) {
394 return new CapturerChoice(capturer, captureFormat);
395 }
396 }
397 return null;
398 }
399
400 public PeerConnection.PeerConnectionState getState() {
401 return requirePeerConnection().connectionState();
402 }
403
404 EglBase.Context getEglBaseContext() {
405 return this.eglBase.getEglBaseContext();
406 }
407
408 public Optional<VideoTrack> getLocalVideoTrack() {
409 return Optional.fromNullable(this.localVideoTrack);
410 }
411
412 public Optional<VideoTrack> getRemoteVideoTrack() {
413 return Optional.fromNullable(this.remoteVideoTrack);
414 }
415
416 private PeerConnection requirePeerConnection() {
417 final PeerConnection peerConnection = this.peerConnection;
418 if (peerConnection == null) {
419 throw new IllegalStateException("initialize PeerConnection first");
420 }
421 return peerConnection;
422 }
423
424 private Context requireContext() {
425 final Context context = this.context;
426 if (context == null) {
427 throw new IllegalStateException("call setup first");
428 }
429 return context;
430 }
431
432 public AppRTCAudioManager getAudioManager() {
433 return appRTCAudioManager;
434 }
435
436 public interface EventCallback {
437 void onIceCandidate(IceCandidate iceCandidate);
438
439 void onConnectionChange(PeerConnection.PeerConnectionState newState);
440
441 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
442 }
443
444 private static abstract class SetSdpObserver implements SdpObserver {
445
446 @Override
447 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
448 throw new IllegalStateException("Not able to use SetSdpObserver");
449 }
450
451 @Override
452 public void onCreateFailure(String s) {
453 throw new IllegalStateException("Not able to use SetSdpObserver");
454 }
455
456 }
457
458 private static abstract class CreateSdpObserver implements SdpObserver {
459
460
461 @Override
462 public void onSetSuccess() {
463 throw new IllegalStateException("Not able to use CreateSdpObserver");
464 }
465
466
467 @Override
468 public void onSetFailure(String s) {
469 throw new IllegalStateException("Not able to use CreateSdpObserver");
470 }
471 }
472
473 public static class InitializationException extends Exception {
474
475 private InitializationException(String message) {
476 super(message);
477 }
478 }
479
480 private static class CapturerChoice {
481 private final CameraVideoCapturer cameraVideoCapturer;
482 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
483
484 public CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat) {
485 this.cameraVideoCapturer = cameraVideoCapturer;
486 this.captureFormat = captureFormat;
487 }
488
489 public int getFrameRate() {
490 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
491 }
492 }
493}