1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerationAndroid;
21import org.webrtc.CameraEnumerator;
22import org.webrtc.CameraVideoCapturer;
23import org.webrtc.CandidatePairChangeEvent;
24import org.webrtc.DataChannel;
25import org.webrtc.DefaultVideoDecoderFactory;
26import org.webrtc.DefaultVideoEncoderFactory;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.MediaConstraints;
30import org.webrtc.MediaStream;
31import org.webrtc.PeerConnection;
32import org.webrtc.PeerConnectionFactory;
33import org.webrtc.RtpReceiver;
34import org.webrtc.SdpObserver;
35import org.webrtc.SessionDescription;
36import org.webrtc.SurfaceTextureHelper;
37import org.webrtc.VideoSource;
38import org.webrtc.VideoTrack;
39
40import java.util.ArrayList;
41import java.util.Collections;
42import java.util.List;
43import java.util.Set;
44
45import javax.annotation.Nonnull;
46import javax.annotation.Nullable;
47
48import eu.siacs.conversations.Config;
49import eu.siacs.conversations.services.AppRTCAudioManager;
50
51public class WebRTCWrapper {
52
53 private static final int CAPTURING_RESOLUTION = 1920;
54 private static final int CAPTURING_MAX_FRAME_RATE = 30;
55
56 private final EventCallback eventCallback;
57 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
58 @Override
59 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
60 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
61 }
62 };
63 private final Handler mainHandler = new Handler(Looper.getMainLooper());
64 private VideoTrack localVideoTrack = null;
65 private VideoTrack remoteVideoTrack = null;
66 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
67 @Override
68 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
69 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
70 //this is called after removeTrack or addTrack
71 //and should then trigger a content-add or content-remove or something
72 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
73 }
74
75 @Override
76 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
77 eventCallback.onConnectionChange(newState);
78 }
79
80 @Override
81 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
82
83 }
84
85 @Override
86 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
87 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
88 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
89 }
90
91 @Override
92 public void onIceConnectionReceivingChange(boolean b) {
93
94 }
95
96 @Override
97 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
98
99 }
100
101 @Override
102 public void onIceCandidate(IceCandidate iceCandidate) {
103 eventCallback.onIceCandidate(iceCandidate);
104 }
105
106 @Override
107 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
108
109 }
110
111 @Override
112 public void onAddStream(MediaStream mediaStream) {
113 Log.d(Config.LOGTAG, "onAddStream");
114 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
115 if (videoTracks.size() > 0) {
116 Log.d(Config.LOGTAG, "more than zero remote video tracks found. using first");
117 remoteVideoTrack = videoTracks.get(0);
118 }
119 }
120
121 @Override
122 public void onRemoveStream(MediaStream mediaStream) {
123
124 }
125
126 @Override
127 public void onDataChannel(DataChannel dataChannel) {
128
129 }
130
131 @Override
132 public void onRenegotiationNeeded() {
133
134 }
135
136 @Override
137 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
138 Log.d(Config.LOGTAG, "onAddTrack()");
139
140 }
141 };
142 @Nullable
143 private PeerConnection peerConnection = null;
144 private AudioTrack localAudioTrack = null;
145 private AppRTCAudioManager appRTCAudioManager = null;
146 private Context context = null;
147 private EglBase eglBase = null;
148 private CapturerChoice capturerChoice;
149
150 public WebRTCWrapper(final EventCallback eventCallback) {
151 this.eventCallback = eventCallback;
152 }
153
154 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
155 PeerConnectionFactory.initialize(
156 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
157 );
158 this.eglBase = EglBase.create();
159 this.context = context;
160 mainHandler.post(() -> {
161 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
162 appRTCAudioManager.start(audioManagerEvents);
163 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
164 });
165 }
166
167 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
168 Preconditions.checkState(this.eglBase != null);
169 Preconditions.checkNotNull(media);
170 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
171 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
172 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
173 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
174 .createPeerConnectionFactory();
175
176
177 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
178
179 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
180
181 if (optionalCapturerChoice.isPresent()) {
182 this.capturerChoice = optionalCapturerChoice.get();
183 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
184 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
185 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
186 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
187 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
188 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
189
190 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
191
192 stream.addTrack(this.localVideoTrack);
193 }
194
195
196 if (media.contains(Media.AUDIO)) {
197 //set up audio track
198 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
199 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
200 stream.addTrack(this.localAudioTrack);
201 }
202
203
204 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
205 if (peerConnection == null) {
206 throw new InitializationException("Unable to create PeerConnection");
207 }
208 peerConnection.addStream(stream);
209 peerConnection.setAudioPlayout(true);
210 peerConnection.setAudioRecording(true);
211 this.peerConnection = peerConnection;
212 }
213
214 public void close() {
215 final PeerConnection peerConnection = this.peerConnection;
216 final CapturerChoice capturerChoice = this.capturerChoice;
217 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
218 final EglBase eglBase = this.eglBase;
219 if (peerConnection != null) {
220 peerConnection.dispose();
221 this.peerConnection = null;
222 }
223 if (audioManager != null) {
224 mainHandler.post(audioManager::stop);
225 }
226 this.localVideoTrack = null;
227 this.remoteVideoTrack = null;
228 if (capturerChoice != null) {
229 try {
230 capturerChoice.cameraVideoCapturer.stopCapture();
231 } catch (InterruptedException e) {
232 Log.e(Config.LOGTAG, "unable to stop capturing");
233 }
234 }
235 if (eglBase != null) {
236 eglBase.release();
237 this.eglBase = null;
238 }
239 }
240
241 void verifyClosed() {
242 if (this.peerConnection != null
243 || this.eglBase != null
244 || this.localVideoTrack != null
245 || this.remoteVideoTrack != null) {
246 throw new IllegalStateException("WebRTCWrapper hasn't been closed properly");
247 }
248 }
249
250 boolean isMicrophoneEnabled() {
251 final AudioTrack audioTrack = this.localAudioTrack;
252 if (audioTrack == null) {
253 throw new IllegalStateException("Local audio track does not exist (yet)");
254 }
255 return audioTrack.enabled();
256 }
257
258 void setMicrophoneEnabled(final boolean enabled) {
259 final AudioTrack audioTrack = this.localAudioTrack;
260 if (audioTrack == null) {
261 throw new IllegalStateException("Local audio track does not exist (yet)");
262 }
263 audioTrack.setEnabled(enabled);
264 }
265
266 public boolean isVideoEnabled() {
267 final VideoTrack videoTrack = this.localVideoTrack;
268 if (videoTrack == null) {
269 throw new IllegalStateException("Local video track does not exist");
270 }
271 return videoTrack.enabled();
272 }
273
274 public void setVideoEnabled(final boolean enabled) {
275 final VideoTrack videoTrack = this.localVideoTrack;
276 if (videoTrack == null) {
277 throw new IllegalStateException("Local video track does not exist");
278 }
279 videoTrack.setEnabled(enabled);
280 }
281
282 public ListenableFuture<SessionDescription> createOffer() {
283 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
284 final SettableFuture<SessionDescription> future = SettableFuture.create();
285 peerConnection.createOffer(new CreateSdpObserver() {
286 @Override
287 public void onCreateSuccess(SessionDescription sessionDescription) {
288 future.set(sessionDescription);
289 }
290
291 @Override
292 public void onCreateFailure(String s) {
293 Log.d(Config.LOGTAG, "create failure" + s);
294 future.setException(new IllegalStateException("Unable to create offer: " + s));
295 }
296 }, new MediaConstraints());
297 return future;
298 }, MoreExecutors.directExecutor());
299 }
300
301 public ListenableFuture<SessionDescription> createAnswer() {
302 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
303 final SettableFuture<SessionDescription> future = SettableFuture.create();
304 peerConnection.createAnswer(new CreateSdpObserver() {
305 @Override
306 public void onCreateSuccess(SessionDescription sessionDescription) {
307 future.set(sessionDescription);
308 }
309
310 @Override
311 public void onCreateFailure(String s) {
312 future.setException(new IllegalStateException("Unable to create answer: " + s));
313 }
314 }, new MediaConstraints());
315 return future;
316 }, MoreExecutors.directExecutor());
317 }
318
319 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
320 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
321 final SettableFuture<Void> future = SettableFuture.create();
322 peerConnection.setLocalDescription(new SetSdpObserver() {
323 @Override
324 public void onSetSuccess() {
325 future.set(null);
326 }
327
328 @Override
329 public void onSetFailure(String s) {
330 Log.d(Config.LOGTAG, "unable to set local " + s);
331 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
332
333 }
334 }, sessionDescription);
335 return future;
336 }, MoreExecutors.directExecutor());
337 }
338
339 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
340 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
341 final SettableFuture<Void> future = SettableFuture.create();
342 peerConnection.setRemoteDescription(new SetSdpObserver() {
343 @Override
344 public void onSetSuccess() {
345 future.set(null);
346 }
347
348 @Override
349 public void onSetFailure(String s) {
350 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
351
352 }
353 }, sessionDescription);
354 return future;
355 }, MoreExecutors.directExecutor());
356 }
357
358 @Nonnull
359 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
360 final PeerConnection peerConnection = this.peerConnection;
361 if (peerConnection == null) {
362 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
363 } else {
364 return Futures.immediateFuture(peerConnection);
365 }
366 }
367
368 public void addIceCandidate(IceCandidate iceCandidate) {
369 requirePeerConnection().addIceCandidate(iceCandidate);
370 }
371
372 private CameraEnumerator getCameraEnumerator() {
373 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
374 return new Camera2Enumerator(requireContext());
375 } else {
376 return new Camera1Enumerator();
377 }
378 }
379
380 private Optional<CapturerChoice> getVideoCapturer() {
381 final CameraEnumerator enumerator = getCameraEnumerator();
382 final String[] deviceNames = enumerator.getDeviceNames();
383 for (final String deviceName : deviceNames) {
384 if (enumerator.isFrontFacing(deviceName)) {
385 return Optional.fromNullable(of(enumerator, deviceName));
386 }
387 }
388 if (deviceNames.length == 0) {
389 return Optional.absent();
390 } else {
391 return Optional.fromNullable(of(enumerator, deviceNames[0]));
392 }
393 }
394
395 @Nullable
396 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName) {
397 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
398 if (capturer == null) {
399 return null;
400 }
401 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
402 Collections.sort(choices, (a, b) -> b.width - a.width);
403 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
404 if (captureFormat.width <= CAPTURING_RESOLUTION) {
405 return new CapturerChoice(capturer, captureFormat);
406 }
407 }
408 return null;
409 }
410
411 public PeerConnection.PeerConnectionState getState() {
412 return requirePeerConnection().connectionState();
413 }
414
415 EglBase.Context getEglBaseContext() {
416 return this.eglBase.getEglBaseContext();
417 }
418
419 public Optional<VideoTrack> getLocalVideoTrack() {
420 return Optional.fromNullable(this.localVideoTrack);
421 }
422
423 public Optional<VideoTrack> getRemoteVideoTrack() {
424 return Optional.fromNullable(this.remoteVideoTrack);
425 }
426
427 private PeerConnection requirePeerConnection() {
428 final PeerConnection peerConnection = this.peerConnection;
429 if (peerConnection == null) {
430 throw new IllegalStateException("initialize PeerConnection first");
431 }
432 return peerConnection;
433 }
434
435 private Context requireContext() {
436 final Context context = this.context;
437 if (context == null) {
438 throw new IllegalStateException("call setup first");
439 }
440 return context;
441 }
442
443 public AppRTCAudioManager getAudioManager() {
444 return appRTCAudioManager;
445 }
446
447 public interface EventCallback {
448 void onIceCandidate(IceCandidate iceCandidate);
449
450 void onConnectionChange(PeerConnection.PeerConnectionState newState);
451
452 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
453 }
454
455 private static abstract class SetSdpObserver implements SdpObserver {
456
457 @Override
458 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
459 throw new IllegalStateException("Not able to use SetSdpObserver");
460 }
461
462 @Override
463 public void onCreateFailure(String s) {
464 throw new IllegalStateException("Not able to use SetSdpObserver");
465 }
466
467 }
468
469 private static abstract class CreateSdpObserver implements SdpObserver {
470
471
472 @Override
473 public void onSetSuccess() {
474 throw new IllegalStateException("Not able to use CreateSdpObserver");
475 }
476
477
478 @Override
479 public void onSetFailure(String s) {
480 throw new IllegalStateException("Not able to use CreateSdpObserver");
481 }
482 }
483
484 public static class InitializationException extends Exception {
485
486 private InitializationException(String message) {
487 super(message);
488 }
489 }
490
491 private static class CapturerChoice {
492 private final CameraVideoCapturer cameraVideoCapturer;
493 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
494
495 public CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat) {
496 this.cameraVideoCapturer = cameraVideoCapturer;
497 this.captureFormat = captureFormat;
498 }
499
500 public int getFrameRate() {
501 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
502 }
503 }
504}