1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerator;
21import org.webrtc.CameraVideoCapturer;
22import org.webrtc.CandidatePairChangeEvent;
23import org.webrtc.DataChannel;
24import org.webrtc.DefaultVideoDecoderFactory;
25import org.webrtc.DefaultVideoEncoderFactory;
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.MediaConstraints;
29import org.webrtc.MediaStream;
30import org.webrtc.PeerConnection;
31import org.webrtc.PeerConnectionFactory;
32import org.webrtc.RtpReceiver;
33import org.webrtc.SdpObserver;
34import org.webrtc.SessionDescription;
35import org.webrtc.SurfaceTextureHelper;
36import org.webrtc.VideoSource;
37import org.webrtc.VideoTrack;
38
39import java.util.List;
40import java.util.Set;
41
42import javax.annotation.Nonnull;
43import javax.annotation.Nullable;
44
45import eu.siacs.conversations.Config;
46import eu.siacs.conversations.services.AppRTCAudioManager;
47
48public class WebRTCWrapper {
49
50 private final EventCallback eventCallback;
51 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
52 @Override
53 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
54 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
55 }
56 };
57 private final Handler mainHandler = new Handler(Looper.getMainLooper());
58 private VideoTrack localVideoTrack = null;
59 private VideoTrack remoteVideoTrack = null;
60 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
61 @Override
62 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
63 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
64 //this is called after removeTrack or addTrack
65 //and should then trigger a content-add or content-remove or something
66 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
67 }
68
69 @Override
70 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
71 eventCallback.onConnectionChange(newState);
72 }
73
74 @Override
75 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
76
77 }
78
79 @Override
80 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
81 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
82 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
83 }
84
85 @Override
86 public void onIceConnectionReceivingChange(boolean b) {
87
88 }
89
90 @Override
91 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
92
93 }
94
95 @Override
96 public void onIceCandidate(IceCandidate iceCandidate) {
97 eventCallback.onIceCandidate(iceCandidate);
98 }
99
100 @Override
101 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
102
103 }
104
105 @Override
106 public void onAddStream(MediaStream mediaStream) {
107 Log.d(Config.LOGTAG, "onAddStream");
108 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
109 if (videoTracks.size() > 0) {
110 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
111 remoteVideoTrack = videoTracks.get(0);
112 }
113 }
114
115 @Override
116 public void onRemoveStream(MediaStream mediaStream) {
117
118 }
119
120 @Override
121 public void onDataChannel(DataChannel dataChannel) {
122
123 }
124
125 @Override
126 public void onRenegotiationNeeded() {
127
128 }
129
130 @Override
131 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
132 Log.d(Config.LOGTAG, "onAddTrack()");
133
134 }
135 };
136 @Nullable
137 private PeerConnection peerConnection = null;
138 private AudioTrack localAudioTrack = null;
139 private AppRTCAudioManager appRTCAudioManager = null;
140 private Context context = null;
141 private EglBase eglBase = null;
142
143 public WebRTCWrapper(final EventCallback eventCallback) {
144 this.eventCallback = eventCallback;
145 }
146
147 public void setup(final Context context) {
148 PeerConnectionFactory.initialize(
149 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
150 );
151 this.eglBase = EglBase.create();
152 this.context = context;
153 mainHandler.post(() -> {
154 appRTCAudioManager = AppRTCAudioManager.create(context, AppRTCAudioManager.SpeakerPhonePreference.EARPIECE);
155 appRTCAudioManager.start(audioManagerEvents);
156 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
157 });
158 }
159
160 public void initializePeerConnection(final List<PeerConnection.IceServer> iceServers) throws InitializationException {
161 Preconditions.checkState(this.eglBase != null);
162 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
163 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
164 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
165 .createPeerConnectionFactory();
166
167
168 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
169
170 final Optional<CameraVideoCapturer> optionalCapturer = getVideoCapturer();
171
172 if (optionalCapturer.isPresent()) {
173 final CameraVideoCapturer capturer = optionalCapturer.get();
174 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
175 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
176 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
177 capturer.startCapture(320, 240, 30);
178
179 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
180
181 stream.addTrack(this.localVideoTrack);
182 }
183
184
185 //set up audio track
186 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
187 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
188 stream.addTrack(this.localAudioTrack);
189
190
191 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
192 if (peerConnection == null) {
193 throw new InitializationException("Unable to create PeerConnection");
194 }
195 peerConnection.addStream(stream);
196 peerConnection.setAudioPlayout(true);
197 peerConnection.setAudioRecording(true);
198 this.peerConnection = peerConnection;
199 }
200
201 public void close() {
202 final PeerConnection peerConnection = this.peerConnection;
203 if (peerConnection != null) {
204 peerConnection.dispose();
205 }
206 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
207 if (audioManager != null) {
208 mainHandler.post(audioManager::stop);
209 }
210 }
211
212 public boolean isMicrophoneEnabled() {
213 final AudioTrack audioTrack = this.localAudioTrack;
214 if (audioTrack == null) {
215 throw new IllegalStateException("Local audio track does not exist (yet)");
216 }
217 return audioTrack.enabled();
218 }
219
220 public void setMicrophoneEnabled(final boolean enabled) {
221 final AudioTrack audioTrack = this.localAudioTrack;
222 if (audioTrack == null) {
223 throw new IllegalStateException("Local audio track does not exist (yet)");
224 }
225 audioTrack.setEnabled(enabled);
226 }
227
228 public ListenableFuture<SessionDescription> createOffer() {
229 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
230 final SettableFuture<SessionDescription> future = SettableFuture.create();
231 peerConnection.createOffer(new CreateSdpObserver() {
232 @Override
233 public void onCreateSuccess(SessionDescription sessionDescription) {
234 future.set(sessionDescription);
235 }
236
237 @Override
238 public void onCreateFailure(String s) {
239 Log.d(Config.LOGTAG, "create failure" + s);
240 future.setException(new IllegalStateException("Unable to create offer: " + s));
241 }
242 }, new MediaConstraints());
243 return future;
244 }, MoreExecutors.directExecutor());
245 }
246
247 public ListenableFuture<SessionDescription> createAnswer() {
248 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
249 final SettableFuture<SessionDescription> future = SettableFuture.create();
250 peerConnection.createAnswer(new CreateSdpObserver() {
251 @Override
252 public void onCreateSuccess(SessionDescription sessionDescription) {
253 future.set(sessionDescription);
254 }
255
256 @Override
257 public void onCreateFailure(String s) {
258 future.setException(new IllegalStateException("Unable to create answer: " + s));
259 }
260 }, new MediaConstraints());
261 return future;
262 }, MoreExecutors.directExecutor());
263 }
264
265 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
266 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
267 final SettableFuture<Void> future = SettableFuture.create();
268 peerConnection.setLocalDescription(new SetSdpObserver() {
269 @Override
270 public void onSetSuccess() {
271 future.set(null);
272 }
273
274 @Override
275 public void onSetFailure(String s) {
276 Log.d(Config.LOGTAG, "unable to set local " + s);
277 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
278
279 }
280 }, sessionDescription);
281 return future;
282 }, MoreExecutors.directExecutor());
283 }
284
285 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
286 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
287 final SettableFuture<Void> future = SettableFuture.create();
288 peerConnection.setRemoteDescription(new SetSdpObserver() {
289 @Override
290 public void onSetSuccess() {
291 future.set(null);
292 }
293
294 @Override
295 public void onSetFailure(String s) {
296 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
297
298 }
299 }, sessionDescription);
300 return future;
301 }, MoreExecutors.directExecutor());
302 }
303
304 @Nonnull
305 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
306 final PeerConnection peerConnection = this.peerConnection;
307 if (peerConnection == null) {
308 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
309 } else {
310 return Futures.immediateFuture(peerConnection);
311 }
312 }
313
314 public void addIceCandidate(IceCandidate iceCandidate) {
315 requirePeerConnection().addIceCandidate(iceCandidate);
316 }
317
318 private CameraEnumerator getCameraEnumerator() {
319 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
320 return new Camera2Enumerator(requireContext());
321 } else {
322 return new Camera1Enumerator();
323 }
324 }
325
326 private Optional<CameraVideoCapturer> getVideoCapturer() {
327 final CameraEnumerator enumerator = getCameraEnumerator();
328 final String[] deviceNames = enumerator.getDeviceNames();
329 for (String deviceName : deviceNames) {
330 if (enumerator.isFrontFacing(deviceName)) {
331 return Optional.fromNullable(enumerator.createCapturer(deviceName, null));
332 }
333 }
334 if (deviceNames.length == 0) {
335 return Optional.absent();
336 } else {
337 return Optional.fromNullable(enumerator.createCapturer(deviceNames[0], null));
338 }
339 }
340
341 public PeerConnection.PeerConnectionState getState() {
342 return requirePeerConnection().connectionState();
343 }
344
345 public EglBase.Context getEglBaseContext() {
346 return this.eglBase.getEglBaseContext();
347 }
348
349 public Optional<VideoTrack> getLocalVideoTrack() {
350 return Optional.fromNullable(this.localVideoTrack);
351 }
352
353 public Optional<VideoTrack> getRemoteVideoTrack() {
354 return Optional.fromNullable(this.remoteVideoTrack);
355 }
356
357 private PeerConnection requirePeerConnection() {
358 final PeerConnection peerConnection = this.peerConnection;
359 if (peerConnection == null) {
360 throw new IllegalStateException("initialize PeerConnection first");
361 }
362 return peerConnection;
363 }
364
365 private Context requireContext() {
366 final Context context = this.context;
367 if (context == null) {
368 throw new IllegalStateException("call setup first");
369 }
370 return context;
371 }
372
373 public AppRTCAudioManager getAudioManager() {
374 return appRTCAudioManager;
375 }
376
377 public interface EventCallback {
378 void onIceCandidate(IceCandidate iceCandidate);
379
380 void onConnectionChange(PeerConnection.PeerConnectionState newState);
381
382 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
383 }
384
385 private static abstract class SetSdpObserver implements SdpObserver {
386
387 @Override
388 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
389 throw new IllegalStateException("Not able to use SetSdpObserver");
390 }
391
392 @Override
393 public void onCreateFailure(String s) {
394 throw new IllegalStateException("Not able to use SetSdpObserver");
395 }
396
397 }
398
399 private static abstract class CreateSdpObserver implements SdpObserver {
400
401
402 @Override
403 public void onSetSuccess() {
404 throw new IllegalStateException("Not able to use CreateSdpObserver");
405 }
406
407
408 @Override
409 public void onSetFailure(String s) {
410 throw new IllegalStateException("Not able to use CreateSdpObserver");
411 }
412 }
413
414 public static class InitializationException extends Exception {
415
416 private InitializationException(String message) {
417 super(message);
418 }
419 }
420}