1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerator;
21import org.webrtc.CameraVideoCapturer;
22import org.webrtc.CandidatePairChangeEvent;
23import org.webrtc.DataChannel;
24import org.webrtc.DefaultVideoDecoderFactory;
25import org.webrtc.DefaultVideoEncoderFactory;
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.MediaConstraints;
29import org.webrtc.MediaStream;
30import org.webrtc.PeerConnection;
31import org.webrtc.PeerConnectionFactory;
32import org.webrtc.RtpReceiver;
33import org.webrtc.SdpObserver;
34import org.webrtc.SessionDescription;
35import org.webrtc.SurfaceTextureHelper;
36import org.webrtc.VideoSource;
37import org.webrtc.VideoTrack;
38
39import java.util.List;
40import java.util.Set;
41
42import javax.annotation.Nonnull;
43import javax.annotation.Nullable;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.services.AppRTCAudioManager;
47
48public class WebRTCWrapper {
49
50 private final EventCallback eventCallback;
51 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
52 @Override
53 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
54 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
55 }
56 };
57 private final Handler mainHandler = new Handler(Looper.getMainLooper());
58 private VideoTrack localVideoTrack = null;
59 private VideoTrack remoteVideoTrack = null;
60 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
61 @Override
62 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
63 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
64 //this is called after removeTrack or addTrack
65 //and should then trigger a content-add or content-remove or something
66 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
67 }
68
69 @Override
70 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
71 eventCallback.onConnectionChange(newState);
72 }
73
74 @Override
75 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
76
77 }
78
79 @Override
80 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
81 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
82 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
83 }
84
85 @Override
86 public void onIceConnectionReceivingChange(boolean b) {
87
88 }
89
90 @Override
91 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
92
93 }
94
95 @Override
96 public void onIceCandidate(IceCandidate iceCandidate) {
97 eventCallback.onIceCandidate(iceCandidate);
98 }
99
100 @Override
101 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
102
103 }
104
105 @Override
106 public void onAddStream(MediaStream mediaStream) {
107 Log.d(Config.LOGTAG, "onAddStream");
108 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
109 if (videoTracks.size() > 0) {
110 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
111 remoteVideoTrack = videoTracks.get(0);
112 }
113 }
114
115 @Override
116 public void onRemoveStream(MediaStream mediaStream) {
117
118 }
119
120 @Override
121 public void onDataChannel(DataChannel dataChannel) {
122
123 }
124
125 @Override
126 public void onRenegotiationNeeded() {
127
128 }
129
130 @Override
131 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
132 Log.d(Config.LOGTAG, "onAddTrack()");
133
134 }
135 };
136 @Nullable
137 private PeerConnection peerConnection = null;
138 private AudioTrack localAudioTrack = null;
139 private AppRTCAudioManager appRTCAudioManager = null;
140 private Context context = null;
141 private EglBase eglBase = null;
142 private Optional<CameraVideoCapturer> optionalCapturer;
143
144 public WebRTCWrapper(final EventCallback eventCallback) {
145 this.eventCallback = eventCallback;
146 }
147
148 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
149 PeerConnectionFactory.initialize(
150 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
151 );
152 this.eglBase = EglBase.create();
153 this.context = context;
154 mainHandler.post(() -> {
155 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
156 appRTCAudioManager.start(audioManagerEvents);
157 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
158 });
159 }
160
161 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
162 Preconditions.checkState(this.eglBase != null);
163 Preconditions.checkNotNull(media);
164 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
165 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
166 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
167 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
168 .createPeerConnectionFactory();
169
170
171 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
172
173 this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
174
175 if (this.optionalCapturer.isPresent()) {
176 final CameraVideoCapturer capturer = this.optionalCapturer.get();
177 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
178 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
179 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
180 capturer.startCapture(320, 240, 30);
181
182 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
183
184 stream.addTrack(this.localVideoTrack);
185 }
186
187
188 if (media.contains(Media.AUDIO)) {
189 //set up audio track
190 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
191 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
192 stream.addTrack(this.localAudioTrack);
193 }
194
195
196 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
197 if (peerConnection == null) {
198 throw new InitializationException("Unable to create PeerConnection");
199 }
200 peerConnection.addStream(stream);
201 peerConnection.setAudioPlayout(true);
202 peerConnection.setAudioRecording(true);
203 this.peerConnection = peerConnection;
204 }
205
206 public void close() {
207 final PeerConnection peerConnection = this.peerConnection;
208 final Optional<CameraVideoCapturer> optionalCapturer = this.optionalCapturer;
209 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
210 final EglBase eglBase = this.eglBase;
211 if (peerConnection != null) {
212 peerConnection.dispose();
213 }
214 if (audioManager != null) {
215 mainHandler.post(audioManager::stop);
216 }
217 this.localVideoTrack = null;
218 this.remoteVideoTrack = null;
219 if (optionalCapturer != null && optionalCapturer.isPresent()) {
220 try {
221 optionalCapturer.get().stopCapture();
222 } catch (InterruptedException e) {
223 Log.e(Config.LOGTAG, "unable to stop capturing");
224 }
225 }
226 if (eglBase != null) {
227 eglBase.release();
228 }
229 }
230
231 boolean isMicrophoneEnabled() {
232 final AudioTrack audioTrack = this.localAudioTrack;
233 if (audioTrack == null) {
234 throw new IllegalStateException("Local audio track does not exist (yet)");
235 }
236 return audioTrack.enabled();
237 }
238
239 void setMicrophoneEnabled(final boolean enabled) {
240 final AudioTrack audioTrack = this.localAudioTrack;
241 if (audioTrack == null) {
242 throw new IllegalStateException("Local audio track does not exist (yet)");
243 }
244 audioTrack.setEnabled(enabled);
245 }
246
247 public boolean isVideoEnabled() {
248 final VideoTrack videoTrack = this.localVideoTrack;
249 if (videoTrack == null) {
250 throw new IllegalStateException("Local video track does not exist");
251 }
252 return videoTrack.enabled();
253 }
254
255 public void setVideoEnabled(final boolean enabled) {
256 final VideoTrack videoTrack = this.localVideoTrack;
257 if (videoTrack == null) {
258 throw new IllegalStateException("Local video track does not exist");
259 }
260 videoTrack.setEnabled(enabled);
261 }
262
263 public ListenableFuture<SessionDescription> createOffer() {
264 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
265 final SettableFuture<SessionDescription> future = SettableFuture.create();
266 peerConnection.createOffer(new CreateSdpObserver() {
267 @Override
268 public void onCreateSuccess(SessionDescription sessionDescription) {
269 future.set(sessionDescription);
270 }
271
272 @Override
273 public void onCreateFailure(String s) {
274 Log.d(Config.LOGTAG, "create failure" + s);
275 future.setException(new IllegalStateException("Unable to create offer: " + s));
276 }
277 }, new MediaConstraints());
278 return future;
279 }, MoreExecutors.directExecutor());
280 }
281
282 public ListenableFuture<SessionDescription> createAnswer() {
283 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
284 final SettableFuture<SessionDescription> future = SettableFuture.create();
285 peerConnection.createAnswer(new CreateSdpObserver() {
286 @Override
287 public void onCreateSuccess(SessionDescription sessionDescription) {
288 future.set(sessionDescription);
289 }
290
291 @Override
292 public void onCreateFailure(String s) {
293 future.setException(new IllegalStateException("Unable to create answer: " + s));
294 }
295 }, new MediaConstraints());
296 return future;
297 }, MoreExecutors.directExecutor());
298 }
299
300 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
301 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
302 final SettableFuture<Void> future = SettableFuture.create();
303 peerConnection.setLocalDescription(new SetSdpObserver() {
304 @Override
305 public void onSetSuccess() {
306 future.set(null);
307 }
308
309 @Override
310 public void onSetFailure(String s) {
311 Log.d(Config.LOGTAG, "unable to set local " + s);
312 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
313
314 }
315 }, sessionDescription);
316 return future;
317 }, MoreExecutors.directExecutor());
318 }
319
320 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
321 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
322 final SettableFuture<Void> future = SettableFuture.create();
323 peerConnection.setRemoteDescription(new SetSdpObserver() {
324 @Override
325 public void onSetSuccess() {
326 future.set(null);
327 }
328
329 @Override
330 public void onSetFailure(String s) {
331 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
332
333 }
334 }, sessionDescription);
335 return future;
336 }, MoreExecutors.directExecutor());
337 }
338
339 @Nonnull
340 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
341 final PeerConnection peerConnection = this.peerConnection;
342 if (peerConnection == null) {
343 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
344 } else {
345 return Futures.immediateFuture(peerConnection);
346 }
347 }
348
349 public void addIceCandidate(IceCandidate iceCandidate) {
350 requirePeerConnection().addIceCandidate(iceCandidate);
351 }
352
353 private CameraEnumerator getCameraEnumerator() {
354 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
355 return new Camera2Enumerator(requireContext());
356 } else {
357 return new Camera1Enumerator();
358 }
359 }
360
361 private Optional<CameraVideoCapturer> getVideoCapturer() {
362 final CameraEnumerator enumerator = getCameraEnumerator();
363 final String[] deviceNames = enumerator.getDeviceNames();
364 for (final String deviceName : deviceNames) {
365 if (enumerator.isFrontFacing(deviceName)) {
366 return Optional.fromNullable(enumerator.createCapturer(deviceName, null));
367 }
368 }
369 if (deviceNames.length == 0) {
370 return Optional.absent();
371 } else {
372 return Optional.fromNullable(enumerator.createCapturer(deviceNames[0], null));
373 }
374 }
375
376 public PeerConnection.PeerConnectionState getState() {
377 return requirePeerConnection().connectionState();
378 }
379
380 public EglBase.Context getEglBaseContext() {
381 return this.eglBase.getEglBaseContext();
382 }
383
384 public Optional<VideoTrack> getLocalVideoTrack() {
385 return Optional.fromNullable(this.localVideoTrack);
386 }
387
388 public Optional<VideoTrack> getRemoteVideoTrack() {
389 return Optional.fromNullable(this.remoteVideoTrack);
390 }
391
392 private PeerConnection requirePeerConnection() {
393 final PeerConnection peerConnection = this.peerConnection;
394 if (peerConnection == null) {
395 throw new IllegalStateException("initialize PeerConnection first");
396 }
397 return peerConnection;
398 }
399
400 private Context requireContext() {
401 final Context context = this.context;
402 if (context == null) {
403 throw new IllegalStateException("call setup first");
404 }
405 return context;
406 }
407
408 public AppRTCAudioManager getAudioManager() {
409 return appRTCAudioManager;
410 }
411
412 public interface EventCallback {
413 void onIceCandidate(IceCandidate iceCandidate);
414
415 void onConnectionChange(PeerConnection.PeerConnectionState newState);
416
417 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
418 }
419
420 private static abstract class SetSdpObserver implements SdpObserver {
421
422 @Override
423 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
424 throw new IllegalStateException("Not able to use SetSdpObserver");
425 }
426
427 @Override
428 public void onCreateFailure(String s) {
429 throw new IllegalStateException("Not able to use SetSdpObserver");
430 }
431
432 }
433
434 private static abstract class CreateSdpObserver implements SdpObserver {
435
436
437 @Override
438 public void onSetSuccess() {
439 throw new IllegalStateException("Not able to use CreateSdpObserver");
440 }
441
442
443 @Override
444 public void onSetFailure(String s) {
445 throw new IllegalStateException("Not able to use CreateSdpObserver");
446 }
447 }
448
449 public static class InitializationException extends Exception {
450
451 private InitializationException(String message) {
452 super(message);
453 }
454 }
455}