1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerator;
21import org.webrtc.CameraVideoCapturer;
22import org.webrtc.CandidatePairChangeEvent;
23import org.webrtc.DataChannel;
24import org.webrtc.DefaultVideoDecoderFactory;
25import org.webrtc.DefaultVideoEncoderFactory;
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.MediaConstraints;
29import org.webrtc.MediaStream;
30import org.webrtc.PeerConnection;
31import org.webrtc.PeerConnectionFactory;
32import org.webrtc.RtpReceiver;
33import org.webrtc.SdpObserver;
34import org.webrtc.SessionDescription;
35import org.webrtc.SurfaceTextureHelper;
36import org.webrtc.VideoSource;
37import org.webrtc.VideoTrack;
38
39import java.util.List;
40import java.util.Set;
41
42import javax.annotation.Nonnull;
43import javax.annotation.Nullable;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.services.AppRTCAudioManager;
47
48public class WebRTCWrapper {
49
50 private final EventCallback eventCallback;
51 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
52 @Override
53 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
54 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
55 }
56 };
57 private final Handler mainHandler = new Handler(Looper.getMainLooper());
58 private VideoTrack localVideoTrack = null;
59 private VideoTrack remoteVideoTrack = null;
60 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
61 @Override
62 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
63 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
64 //this is called after removeTrack or addTrack
65 //and should then trigger a content-add or content-remove or something
66 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
67 }
68
69 @Override
70 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
71 eventCallback.onConnectionChange(newState);
72 }
73
74 @Override
75 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
76
77 }
78
79 @Override
80 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
81 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
82 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
83 }
84
85 @Override
86 public void onIceConnectionReceivingChange(boolean b) {
87
88 }
89
90 @Override
91 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
92
93 }
94
95 @Override
96 public void onIceCandidate(IceCandidate iceCandidate) {
97 eventCallback.onIceCandidate(iceCandidate);
98 }
99
100 @Override
101 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
102
103 }
104
105 @Override
106 public void onAddStream(MediaStream mediaStream) {
107 Log.d(Config.LOGTAG, "onAddStream");
108 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
109 if (videoTracks.size() > 0) {
110 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
111 remoteVideoTrack = videoTracks.get(0);
112 }
113 }
114
115 @Override
116 public void onRemoveStream(MediaStream mediaStream) {
117
118 }
119
120 @Override
121 public void onDataChannel(DataChannel dataChannel) {
122
123 }
124
125 @Override
126 public void onRenegotiationNeeded() {
127
128 }
129
130 @Override
131 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
132 Log.d(Config.LOGTAG, "onAddTrack()");
133
134 }
135 };
136 @Nullable
137 private PeerConnection peerConnection = null;
138 private AudioTrack localAudioTrack = null;
139 private AppRTCAudioManager appRTCAudioManager = null;
140 private Context context = null;
141 private EglBase eglBase = null;
142 private Optional<CameraVideoCapturer> optionalCapturer;
143
144 public WebRTCWrapper(final EventCallback eventCallback) {
145 this.eventCallback = eventCallback;
146 }
147
148 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
149 PeerConnectionFactory.initialize(
150 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
151 );
152 this.eglBase = EglBase.create();
153 this.context = context;
154 mainHandler.post(() -> {
155 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
156 appRTCAudioManager.start(audioManagerEvents);
157 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
158 });
159 }
160
161 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
162 Preconditions.checkState(this.eglBase != null);
163 Preconditions.checkNotNull(media);
164 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
165 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
166 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
167 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
168 .createPeerConnectionFactory();
169
170
171 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
172
173 this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
174
175 if (this.optionalCapturer.isPresent()) {
176 final CameraVideoCapturer capturer = this.optionalCapturer.get();
177 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
178 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
179 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
180 capturer.startCapture(320, 240, 30);
181
182 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
183
184 stream.addTrack(this.localVideoTrack);
185 }
186
187
188 if (media.contains(Media.AUDIO)) {
189 //set up audio track
190 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
191 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
192 stream.addTrack(this.localAudioTrack);
193 }
194
195
196 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
197 if (peerConnection == null) {
198 throw new InitializationException("Unable to create PeerConnection");
199 }
200 peerConnection.addStream(stream);
201 peerConnection.setAudioPlayout(true);
202 peerConnection.setAudioRecording(true);
203 this.peerConnection = peerConnection;
204 }
205
206 public void close() {
207 final PeerConnection peerConnection = this.peerConnection;
208 final Optional<CameraVideoCapturer> optionalCapturer = this.optionalCapturer;
209 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
210 final EglBase eglBase = this.eglBase;
211 if (peerConnection != null) {
212 peerConnection.dispose();
213 }
214 if (audioManager != null) {
215 mainHandler.post(audioManager::stop);
216 }
217 this.localVideoTrack = null;
218 this.remoteVideoTrack = null;
219 if (optionalCapturer != null && optionalCapturer.isPresent()) {
220 try {
221 optionalCapturer.get().stopCapture();
222 } catch (InterruptedException e) {
223 Log.e(Config.LOGTAG, "unable to stop capturing");
224 }
225 }
226 if (eglBase != null) {
227 eglBase.release();
228 }
229 }
230
231 public boolean isMicrophoneEnabled() {
232 final AudioTrack audioTrack = this.localAudioTrack;
233 if (audioTrack == null) {
234 throw new IllegalStateException("Local audio track does not exist (yet)");
235 }
236 return audioTrack.enabled();
237 }
238
239 public void setMicrophoneEnabled(final boolean enabled) {
240 final AudioTrack audioTrack = this.localAudioTrack;
241 if (audioTrack == null) {
242 throw new IllegalStateException("Local audio track does not exist (yet)");
243 }
244 audioTrack.setEnabled(enabled);
245 }
246
247 public ListenableFuture<SessionDescription> createOffer() {
248 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
249 final SettableFuture<SessionDescription> future = SettableFuture.create();
250 peerConnection.createOffer(new CreateSdpObserver() {
251 @Override
252 public void onCreateSuccess(SessionDescription sessionDescription) {
253 future.set(sessionDescription);
254 }
255
256 @Override
257 public void onCreateFailure(String s) {
258 Log.d(Config.LOGTAG, "create failure" + s);
259 future.setException(new IllegalStateException("Unable to create offer: " + s));
260 }
261 }, new MediaConstraints());
262 return future;
263 }, MoreExecutors.directExecutor());
264 }
265
266 public ListenableFuture<SessionDescription> createAnswer() {
267 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
268 final SettableFuture<SessionDescription> future = SettableFuture.create();
269 peerConnection.createAnswer(new CreateSdpObserver() {
270 @Override
271 public void onCreateSuccess(SessionDescription sessionDescription) {
272 future.set(sessionDescription);
273 }
274
275 @Override
276 public void onCreateFailure(String s) {
277 future.setException(new IllegalStateException("Unable to create answer: " + s));
278 }
279 }, new MediaConstraints());
280 return future;
281 }, MoreExecutors.directExecutor());
282 }
283
284 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
285 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
286 final SettableFuture<Void> future = SettableFuture.create();
287 peerConnection.setLocalDescription(new SetSdpObserver() {
288 @Override
289 public void onSetSuccess() {
290 future.set(null);
291 }
292
293 @Override
294 public void onSetFailure(String s) {
295 Log.d(Config.LOGTAG, "unable to set local " + s);
296 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
297
298 }
299 }, sessionDescription);
300 return future;
301 }, MoreExecutors.directExecutor());
302 }
303
304 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
305 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
306 final SettableFuture<Void> future = SettableFuture.create();
307 peerConnection.setRemoteDescription(new SetSdpObserver() {
308 @Override
309 public void onSetSuccess() {
310 future.set(null);
311 }
312
313 @Override
314 public void onSetFailure(String s) {
315 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
316
317 }
318 }, sessionDescription);
319 return future;
320 }, MoreExecutors.directExecutor());
321 }
322
323 @Nonnull
324 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
325 final PeerConnection peerConnection = this.peerConnection;
326 if (peerConnection == null) {
327 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
328 } else {
329 return Futures.immediateFuture(peerConnection);
330 }
331 }
332
333 public void addIceCandidate(IceCandidate iceCandidate) {
334 requirePeerConnection().addIceCandidate(iceCandidate);
335 }
336
337 private CameraEnumerator getCameraEnumerator() {
338 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
339 return new Camera2Enumerator(requireContext());
340 } else {
341 return new Camera1Enumerator();
342 }
343 }
344
345 private Optional<CameraVideoCapturer> getVideoCapturer() {
346 final CameraEnumerator enumerator = getCameraEnumerator();
347 final String[] deviceNames = enumerator.getDeviceNames();
348 for (final String deviceName : deviceNames) {
349 if (enumerator.isFrontFacing(deviceName)) {
350 return Optional.fromNullable(enumerator.createCapturer(deviceName, null));
351 }
352 }
353 if (deviceNames.length == 0) {
354 return Optional.absent();
355 } else {
356 return Optional.fromNullable(enumerator.createCapturer(deviceNames[0], null));
357 }
358 }
359
360 public PeerConnection.PeerConnectionState getState() {
361 return requirePeerConnection().connectionState();
362 }
363
364 public EglBase.Context getEglBaseContext() {
365 return this.eglBase.getEglBaseContext();
366 }
367
368 public Optional<VideoTrack> getLocalVideoTrack() {
369 return Optional.fromNullable(this.localVideoTrack);
370 }
371
372 public Optional<VideoTrack> getRemoteVideoTrack() {
373 return Optional.fromNullable(this.remoteVideoTrack);
374 }
375
376 private PeerConnection requirePeerConnection() {
377 final PeerConnection peerConnection = this.peerConnection;
378 if (peerConnection == null) {
379 throw new IllegalStateException("initialize PeerConnection first");
380 }
381 return peerConnection;
382 }
383
384 private Context requireContext() {
385 final Context context = this.context;
386 if (context == null) {
387 throw new IllegalStateException("call setup first");
388 }
389 return context;
390 }
391
392 public AppRTCAudioManager getAudioManager() {
393 return appRTCAudioManager;
394 }
395
396 public interface EventCallback {
397 void onIceCandidate(IceCandidate iceCandidate);
398
399 void onConnectionChange(PeerConnection.PeerConnectionState newState);
400
401 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
402 }
403
404 private static abstract class SetSdpObserver implements SdpObserver {
405
406 @Override
407 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
408 throw new IllegalStateException("Not able to use SetSdpObserver");
409 }
410
411 @Override
412 public void onCreateFailure(String s) {
413 throw new IllegalStateException("Not able to use SetSdpObserver");
414 }
415
416 }
417
418 private static abstract class CreateSdpObserver implements SdpObserver {
419
420
421 @Override
422 public void onSetSuccess() {
423 throw new IllegalStateException("Not able to use CreateSdpObserver");
424 }
425
426
427 @Override
428 public void onSetFailure(String s) {
429 throw new IllegalStateException("Not able to use CreateSdpObserver");
430 }
431 }
432
433 public static class InitializationException extends Exception {
434
435 private InitializationException(String message) {
436 super(message);
437 }
438 }
439}