1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerationAndroid;
21import org.webrtc.CameraEnumerator;
22import org.webrtc.CameraVideoCapturer;
23import org.webrtc.CandidatePairChangeEvent;
24import org.webrtc.DataChannel;
25import org.webrtc.DefaultVideoDecoderFactory;
26import org.webrtc.DefaultVideoEncoderFactory;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.MediaConstraints;
30import org.webrtc.MediaStream;
31import org.webrtc.PeerConnection;
32import org.webrtc.PeerConnectionFactory;
33import org.webrtc.RtpReceiver;
34import org.webrtc.SdpObserver;
35import org.webrtc.SessionDescription;
36import org.webrtc.SurfaceTextureHelper;
37import org.webrtc.VideoSource;
38import org.webrtc.VideoTrack;
39
40import java.util.ArrayList;
41import java.util.Collections;
42import java.util.Comparator;
43import java.util.List;
44import java.util.Set;
45
46import javax.annotation.Nonnull;
47import javax.annotation.Nullable;
48
49import eu.siacs.conversations.Config;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51
52public class WebRTCWrapper {
53
54 private static final int CAPTURING_RESOLUTION = 1920;
55 private static final int CAPTURING_MAX_FRAME_RATE = 30;
56
57 private final EventCallback eventCallback;
58 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
59 @Override
60 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
61 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
62 }
63 };
64 private final Handler mainHandler = new Handler(Looper.getMainLooper());
65 private VideoTrack localVideoTrack = null;
66 private VideoTrack remoteVideoTrack = null;
67 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
68 @Override
69 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
70 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
71 //this is called after removeTrack or addTrack
72 //and should then trigger a content-add or content-remove or something
73 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
74 }
75
76 @Override
77 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
78 eventCallback.onConnectionChange(newState);
79 }
80
81 @Override
82 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
83
84 }
85
86 @Override
87 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
88 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
89 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
90 }
91
92 @Override
93 public void onIceConnectionReceivingChange(boolean b) {
94
95 }
96
97 @Override
98 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
99
100 }
101
102 @Override
103 public void onIceCandidate(IceCandidate iceCandidate) {
104 eventCallback.onIceCandidate(iceCandidate);
105 }
106
107 @Override
108 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
109
110 }
111
112 @Override
113 public void onAddStream(MediaStream mediaStream) {
114 Log.d(Config.LOGTAG, "onAddStream");
115 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
116 if (videoTracks.size() > 0) {
117 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
118 remoteVideoTrack = videoTracks.get(0);
119 }
120 }
121
122 @Override
123 public void onRemoveStream(MediaStream mediaStream) {
124
125 }
126
127 @Override
128 public void onDataChannel(DataChannel dataChannel) {
129
130 }
131
132 @Override
133 public void onRenegotiationNeeded() {
134
135 }
136
137 @Override
138 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
139 Log.d(Config.LOGTAG, "onAddTrack()");
140
141 }
142 };
143 @Nullable
144 private PeerConnection peerConnection = null;
145 private AudioTrack localAudioTrack = null;
146 private AppRTCAudioManager appRTCAudioManager = null;
147 private Context context = null;
148 private EglBase eglBase = null;
149 private Optional<CapturerChoice> optionalCapturer;
150
151 public WebRTCWrapper(final EventCallback eventCallback) {
152 this.eventCallback = eventCallback;
153 }
154
155 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
156 PeerConnectionFactory.initialize(
157 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
158 );
159 this.eglBase = EglBase.create();
160 this.context = context;
161 mainHandler.post(() -> {
162 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
163 appRTCAudioManager.start(audioManagerEvents);
164 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
165 });
166 }
167
168 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
169 Preconditions.checkState(this.eglBase != null);
170 Preconditions.checkNotNull(media);
171 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
172 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
173 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
174 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
175 .createPeerConnectionFactory();
176
177
178 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
179
180 this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
181
182 if (this.optionalCapturer.isPresent()) {
183 final CapturerChoice choice = this.optionalCapturer.get();
184 final CameraVideoCapturer capturer = choice.cameraVideoCapturer;
185 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
186 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
187 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
188 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", choice.captureFormat.width, choice.captureFormat.height, choice.getFrameRate()));
189 capturer.startCapture(choice.captureFormat.width, choice.captureFormat.height, choice.getFrameRate());
190
191 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
192
193 stream.addTrack(this.localVideoTrack);
194 }
195
196
197 if (media.contains(Media.AUDIO)) {
198 //set up audio track
199 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
200 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
201 stream.addTrack(this.localAudioTrack);
202 }
203
204
205 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
206 if (peerConnection == null) {
207 throw new InitializationException("Unable to create PeerConnection");
208 }
209 peerConnection.addStream(stream);
210 peerConnection.setAudioPlayout(true);
211 peerConnection.setAudioRecording(true);
212 this.peerConnection = peerConnection;
213 }
214
215 public void close() {
216 final PeerConnection peerConnection = this.peerConnection;
217 final Optional<CapturerChoice> optionalCapturer = this.optionalCapturer;
218 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
219 final EglBase eglBase = this.eglBase;
220 if (peerConnection != null) {
221 peerConnection.dispose();
222 }
223 if (audioManager != null) {
224 mainHandler.post(audioManager::stop);
225 }
226 this.localVideoTrack = null;
227 this.remoteVideoTrack = null;
228 if (optionalCapturer != null && optionalCapturer.isPresent()) {
229 try {
230 optionalCapturer.get().cameraVideoCapturer.stopCapture();
231 } catch (InterruptedException e) {
232 Log.e(Config.LOGTAG, "unable to stop capturing");
233 }
234 }
235 if (eglBase != null) {
236 eglBase.release();
237 }
238 }
239
240 boolean isMicrophoneEnabled() {
241 final AudioTrack audioTrack = this.localAudioTrack;
242 if (audioTrack == null) {
243 throw new IllegalStateException("Local audio track does not exist (yet)");
244 }
245 return audioTrack.enabled();
246 }
247
248 void setMicrophoneEnabled(final boolean enabled) {
249 final AudioTrack audioTrack = this.localAudioTrack;
250 if (audioTrack == null) {
251 throw new IllegalStateException("Local audio track does not exist (yet)");
252 }
253 audioTrack.setEnabled(enabled);
254 }
255
256 public boolean isVideoEnabled() {
257 final VideoTrack videoTrack = this.localVideoTrack;
258 if (videoTrack == null) {
259 throw new IllegalStateException("Local video track does not exist");
260 }
261 return videoTrack.enabled();
262 }
263
264 public void setVideoEnabled(final boolean enabled) {
265 final VideoTrack videoTrack = this.localVideoTrack;
266 if (videoTrack == null) {
267 throw new IllegalStateException("Local video track does not exist");
268 }
269 videoTrack.setEnabled(enabled);
270 }
271
272 public ListenableFuture<SessionDescription> createOffer() {
273 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
274 final SettableFuture<SessionDescription> future = SettableFuture.create();
275 peerConnection.createOffer(new CreateSdpObserver() {
276 @Override
277 public void onCreateSuccess(SessionDescription sessionDescription) {
278 future.set(sessionDescription);
279 }
280
281 @Override
282 public void onCreateFailure(String s) {
283 Log.d(Config.LOGTAG, "create failure" + s);
284 future.setException(new IllegalStateException("Unable to create offer: " + s));
285 }
286 }, new MediaConstraints());
287 return future;
288 }, MoreExecutors.directExecutor());
289 }
290
291 public ListenableFuture<SessionDescription> createAnswer() {
292 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
293 final SettableFuture<SessionDescription> future = SettableFuture.create();
294 peerConnection.createAnswer(new CreateSdpObserver() {
295 @Override
296 public void onCreateSuccess(SessionDescription sessionDescription) {
297 future.set(sessionDescription);
298 }
299
300 @Override
301 public void onCreateFailure(String s) {
302 future.setException(new IllegalStateException("Unable to create answer: " + s));
303 }
304 }, new MediaConstraints());
305 return future;
306 }, MoreExecutors.directExecutor());
307 }
308
309 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
310 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
311 final SettableFuture<Void> future = SettableFuture.create();
312 peerConnection.setLocalDescription(new SetSdpObserver() {
313 @Override
314 public void onSetSuccess() {
315 future.set(null);
316 }
317
318 @Override
319 public void onSetFailure(String s) {
320 Log.d(Config.LOGTAG, "unable to set local " + s);
321 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
322
323 }
324 }, sessionDescription);
325 return future;
326 }, MoreExecutors.directExecutor());
327 }
328
329 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
330 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
331 final SettableFuture<Void> future = SettableFuture.create();
332 peerConnection.setRemoteDescription(new SetSdpObserver() {
333 @Override
334 public void onSetSuccess() {
335 future.set(null);
336 }
337
338 @Override
339 public void onSetFailure(String s) {
340 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
341
342 }
343 }, sessionDescription);
344 return future;
345 }, MoreExecutors.directExecutor());
346 }
347
348 @Nonnull
349 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
350 final PeerConnection peerConnection = this.peerConnection;
351 if (peerConnection == null) {
352 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
353 } else {
354 return Futures.immediateFuture(peerConnection);
355 }
356 }
357
358 public void addIceCandidate(IceCandidate iceCandidate) {
359 requirePeerConnection().addIceCandidate(iceCandidate);
360 }
361
362 private CameraEnumerator getCameraEnumerator() {
363 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
364 return new Camera2Enumerator(requireContext());
365 } else {
366 return new Camera1Enumerator();
367 }
368 }
369
370 private Optional<CapturerChoice> getVideoCapturer() {
371 final CameraEnumerator enumerator = getCameraEnumerator();
372 final String[] deviceNames = enumerator.getDeviceNames();
373 for (final String deviceName : deviceNames) {
374 if (enumerator.isFrontFacing(deviceName)) {
375 return Optional.fromNullable(of(enumerator, deviceName));
376 }
377 }
378 if (deviceNames.length == 0) {
379 return Optional.absent();
380 } else {
381 return Optional.fromNullable(of(enumerator, deviceNames[0]));
382 }
383 }
384
385 @Nullable
386 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName) {
387 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
388 if (capturer == null) {
389 return null;
390 }
391 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
392 Collections.sort(choices, (a, b) -> b.width - a.width);
393 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
394 if (captureFormat.width <= CAPTURING_RESOLUTION) {
395 return new CapturerChoice(capturer, captureFormat);
396 }
397 }
398 return null;
399 }
400
401 public PeerConnection.PeerConnectionState getState() {
402 return requirePeerConnection().connectionState();
403 }
404
405 EglBase.Context getEglBaseContext() {
406 return this.eglBase.getEglBaseContext();
407 }
408
409 public Optional<VideoTrack> getLocalVideoTrack() {
410 return Optional.fromNullable(this.localVideoTrack);
411 }
412
413 public Optional<VideoTrack> getRemoteVideoTrack() {
414 return Optional.fromNullable(this.remoteVideoTrack);
415 }
416
417 private PeerConnection requirePeerConnection() {
418 final PeerConnection peerConnection = this.peerConnection;
419 if (peerConnection == null) {
420 throw new IllegalStateException("initialize PeerConnection first");
421 }
422 return peerConnection;
423 }
424
425 private Context requireContext() {
426 final Context context = this.context;
427 if (context == null) {
428 throw new IllegalStateException("call setup first");
429 }
430 return context;
431 }
432
433 public AppRTCAudioManager getAudioManager() {
434 return appRTCAudioManager;
435 }
436
437 public interface EventCallback {
438 void onIceCandidate(IceCandidate iceCandidate);
439
440 void onConnectionChange(PeerConnection.PeerConnectionState newState);
441
442 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
443 }
444
445 private static abstract class SetSdpObserver implements SdpObserver {
446
447 @Override
448 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
449 throw new IllegalStateException("Not able to use SetSdpObserver");
450 }
451
452 @Override
453 public void onCreateFailure(String s) {
454 throw new IllegalStateException("Not able to use SetSdpObserver");
455 }
456
457 }
458
459 private static abstract class CreateSdpObserver implements SdpObserver {
460
461
462 @Override
463 public void onSetSuccess() {
464 throw new IllegalStateException("Not able to use CreateSdpObserver");
465 }
466
467
468 @Override
469 public void onSetFailure(String s) {
470 throw new IllegalStateException("Not able to use CreateSdpObserver");
471 }
472 }
473
474 public static class InitializationException extends Exception {
475
476 private InitializationException(String message) {
477 super(message);
478 }
479 }
480
481 private static class CapturerChoice {
482 private final CameraVideoCapturer cameraVideoCapturer;
483 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
484
485 public CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat) {
486 this.cameraVideoCapturer = cameraVideoCapturer;
487 this.captureFormat = captureFormat;
488 }
489
490 public int getFrameRate() {
491 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
492 }
493 }
494}