1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerator;
21import org.webrtc.CameraVideoCapturer;
22import org.webrtc.CandidatePairChangeEvent;
23import org.webrtc.DataChannel;
24import org.webrtc.DefaultVideoDecoderFactory;
25import org.webrtc.DefaultVideoEncoderFactory;
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.MediaConstraints;
29import org.webrtc.MediaStream;
30import org.webrtc.PeerConnection;
31import org.webrtc.PeerConnectionFactory;
32import org.webrtc.RtpReceiver;
33import org.webrtc.SdpObserver;
34import org.webrtc.SessionDescription;
35import org.webrtc.SurfaceTextureHelper;
36import org.webrtc.VideoSource;
37import org.webrtc.VideoTrack;
38
39import java.util.List;
40import java.util.Set;
41
42import javax.annotation.Nonnull;
43import javax.annotation.Nullable;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.services.AppRTCAudioManager;
47
48public class WebRTCWrapper {
49
50 private final EventCallback eventCallback;
51 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
52 @Override
53 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
54 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
55 }
56 };
57 private final Handler mainHandler = new Handler(Looper.getMainLooper());
58 private VideoTrack localVideoTrack = null;
59 private VideoTrack remoteVideoTrack = null;
60 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
61 @Override
62 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
63 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
64 //this is called after removeTrack or addTrack
65 //and should then trigger a content-add or content-remove or something
66 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
67 }
68
69 @Override
70 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
71 eventCallback.onConnectionChange(newState);
72 }
73
74 @Override
75 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
76
77 }
78
79 @Override
80 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
81 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
82 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
83 }
84
85 @Override
86 public void onIceConnectionReceivingChange(boolean b) {
87
88 }
89
90 @Override
91 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
92
93 }
94
95 @Override
96 public void onIceCandidate(IceCandidate iceCandidate) {
97 eventCallback.onIceCandidate(iceCandidate);
98 }
99
100 @Override
101 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
102
103 }
104
105 @Override
106 public void onAddStream(MediaStream mediaStream) {
107 Log.d(Config.LOGTAG, "onAddStream");
108 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
109 if (videoTracks.size() > 0) {
110 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
111 remoteVideoTrack = videoTracks.get(0);
112 }
113 }
114
115 @Override
116 public void onRemoveStream(MediaStream mediaStream) {
117
118 }
119
120 @Override
121 public void onDataChannel(DataChannel dataChannel) {
122
123 }
124
125 @Override
126 public void onRenegotiationNeeded() {
127
128 }
129
130 @Override
131 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
132 Log.d(Config.LOGTAG, "onAddTrack()");
133
134 }
135 };
136 @Nullable
137 private PeerConnection peerConnection = null;
138 private AudioTrack localAudioTrack = null;
139 private AppRTCAudioManager appRTCAudioManager = null;
140 private Context context = null;
141 private EglBase eglBase = null;
142 private Optional<CameraVideoCapturer> optionalCapturer;
143
144 public WebRTCWrapper(final EventCallback eventCallback) {
145 this.eventCallback = eventCallback;
146 }
147
148 public void setup(final Context context) {
149 PeerConnectionFactory.initialize(
150 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
151 );
152 this.eglBase = EglBase.create();
153 this.context = context;
154 mainHandler.post(() -> {
155 appRTCAudioManager = AppRTCAudioManager.create(context, AppRTCAudioManager.SpeakerPhonePreference.EARPIECE);
156 appRTCAudioManager.start(audioManagerEvents);
157 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
158 });
159 }
160
161 public void initializePeerConnection(final List<PeerConnection.IceServer> iceServers) throws InitializationException {
162 Preconditions.checkState(this.eglBase != null);
163 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
164 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
165 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
166 .createPeerConnectionFactory();
167
168
169 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
170
171 this.optionalCapturer = getVideoCapturer();
172
173 if (this.optionalCapturer.isPresent()) {
174 final CameraVideoCapturer capturer = this.optionalCapturer.get();
175 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
176 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
177 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
178 capturer.startCapture(320, 240, 30);
179
180 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
181
182 stream.addTrack(this.localVideoTrack);
183 }
184
185
186 //set up audio track
187 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
188 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
189 stream.addTrack(this.localAudioTrack);
190
191
192 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
193 if (peerConnection == null) {
194 throw new InitializationException("Unable to create PeerConnection");
195 }
196 peerConnection.addStream(stream);
197 peerConnection.setAudioPlayout(true);
198 peerConnection.setAudioRecording(true);
199 this.peerConnection = peerConnection;
200 }
201
202 public void close() {
203 final PeerConnection peerConnection = this.peerConnection;
204 if (peerConnection != null) {
205 peerConnection.dispose();
206 }
207 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
208 if (audioManager != null) {
209 mainHandler.post(audioManager::stop);
210 }
211 this.localVideoTrack = null;
212 this.remoteVideoTrack = null;
213 if (this.optionalCapturer.isPresent()) {
214 try {
215 this.optionalCapturer.get().stopCapture();
216 } catch (InterruptedException e) {
217 Log.e(Config.LOGTAG,"unable to stop capturing");
218 }
219 }
220 eglBase.release();
221 }
222
223 public boolean isMicrophoneEnabled() {
224 final AudioTrack audioTrack = this.localAudioTrack;
225 if (audioTrack == null) {
226 throw new IllegalStateException("Local audio track does not exist (yet)");
227 }
228 return audioTrack.enabled();
229 }
230
231 public void setMicrophoneEnabled(final boolean enabled) {
232 final AudioTrack audioTrack = this.localAudioTrack;
233 if (audioTrack == null) {
234 throw new IllegalStateException("Local audio track does not exist (yet)");
235 }
236 audioTrack.setEnabled(enabled);
237 }
238
239 public ListenableFuture<SessionDescription> createOffer() {
240 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
241 final SettableFuture<SessionDescription> future = SettableFuture.create();
242 peerConnection.createOffer(new CreateSdpObserver() {
243 @Override
244 public void onCreateSuccess(SessionDescription sessionDescription) {
245 future.set(sessionDescription);
246 }
247
248 @Override
249 public void onCreateFailure(String s) {
250 Log.d(Config.LOGTAG, "create failure" + s);
251 future.setException(new IllegalStateException("Unable to create offer: " + s));
252 }
253 }, new MediaConstraints());
254 return future;
255 }, MoreExecutors.directExecutor());
256 }
257
258 public ListenableFuture<SessionDescription> createAnswer() {
259 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
260 final SettableFuture<SessionDescription> future = SettableFuture.create();
261 peerConnection.createAnswer(new CreateSdpObserver() {
262 @Override
263 public void onCreateSuccess(SessionDescription sessionDescription) {
264 future.set(sessionDescription);
265 }
266
267 @Override
268 public void onCreateFailure(String s) {
269 future.setException(new IllegalStateException("Unable to create answer: " + s));
270 }
271 }, new MediaConstraints());
272 return future;
273 }, MoreExecutors.directExecutor());
274 }
275
276 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
277 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
278 final SettableFuture<Void> future = SettableFuture.create();
279 peerConnection.setLocalDescription(new SetSdpObserver() {
280 @Override
281 public void onSetSuccess() {
282 future.set(null);
283 }
284
285 @Override
286 public void onSetFailure(String s) {
287 Log.d(Config.LOGTAG, "unable to set local " + s);
288 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
289
290 }
291 }, sessionDescription);
292 return future;
293 }, MoreExecutors.directExecutor());
294 }
295
296 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
297 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
298 final SettableFuture<Void> future = SettableFuture.create();
299 peerConnection.setRemoteDescription(new SetSdpObserver() {
300 @Override
301 public void onSetSuccess() {
302 future.set(null);
303 }
304
305 @Override
306 public void onSetFailure(String s) {
307 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
308
309 }
310 }, sessionDescription);
311 return future;
312 }, MoreExecutors.directExecutor());
313 }
314
315 @Nonnull
316 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
317 final PeerConnection peerConnection = this.peerConnection;
318 if (peerConnection == null) {
319 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
320 } else {
321 return Futures.immediateFuture(peerConnection);
322 }
323 }
324
325 public void addIceCandidate(IceCandidate iceCandidate) {
326 requirePeerConnection().addIceCandidate(iceCandidate);
327 }
328
329 private CameraEnumerator getCameraEnumerator() {
330 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
331 return new Camera2Enumerator(requireContext());
332 } else {
333 return new Camera1Enumerator();
334 }
335 }
336
337 private Optional<CameraVideoCapturer> getVideoCapturer() {
338 final CameraEnumerator enumerator = getCameraEnumerator();
339 final String[] deviceNames = enumerator.getDeviceNames();
340 for (final String deviceName : deviceNames) {
341 if (enumerator.isFrontFacing(deviceName)) {
342 return Optional.fromNullable(enumerator.createCapturer(deviceName, null));
343 }
344 }
345 if (deviceNames.length == 0) {
346 return Optional.absent();
347 } else {
348 return Optional.fromNullable(enumerator.createCapturer(deviceNames[0], null));
349 }
350 }
351
352 public PeerConnection.PeerConnectionState getState() {
353 return requirePeerConnection().connectionState();
354 }
355
356 public EglBase.Context getEglBaseContext() {
357 return this.eglBase.getEglBaseContext();
358 }
359
360 public Optional<VideoTrack> getLocalVideoTrack() {
361 return Optional.fromNullable(this.localVideoTrack);
362 }
363
364 public Optional<VideoTrack> getRemoteVideoTrack() {
365 return Optional.fromNullable(this.remoteVideoTrack);
366 }
367
368 private PeerConnection requirePeerConnection() {
369 final PeerConnection peerConnection = this.peerConnection;
370 if (peerConnection == null) {
371 throw new IllegalStateException("initialize PeerConnection first");
372 }
373 return peerConnection;
374 }
375
376 private Context requireContext() {
377 final Context context = this.context;
378 if (context == null) {
379 throw new IllegalStateException("call setup first");
380 }
381 return context;
382 }
383
384 public AppRTCAudioManager getAudioManager() {
385 return appRTCAudioManager;
386 }
387
388 public interface EventCallback {
389 void onIceCandidate(IceCandidate iceCandidate);
390
391 void onConnectionChange(PeerConnection.PeerConnectionState newState);
392
393 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
394 }
395
396 private static abstract class SetSdpObserver implements SdpObserver {
397
398 @Override
399 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
400 throw new IllegalStateException("Not able to use SetSdpObserver");
401 }
402
403 @Override
404 public void onCreateFailure(String s) {
405 throw new IllegalStateException("Not able to use SetSdpObserver");
406 }
407
408 }
409
410 private static abstract class CreateSdpObserver implements SdpObserver {
411
412
413 @Override
414 public void onSetSuccess() {
415 throw new IllegalStateException("Not able to use CreateSdpObserver");
416 }
417
418
419 @Override
420 public void onSetFailure(String s) {
421 throw new IllegalStateException("Not able to use CreateSdpObserver");
422 }
423 }
424
425 public static class InitializationException extends Exception {
426
427 private InitializationException(String message) {
428 super(message);
429 }
430 }
431}