1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerationAndroid;
21import org.webrtc.CameraEnumerator;
22import org.webrtc.CameraVideoCapturer;
23import org.webrtc.CandidatePairChangeEvent;
24import org.webrtc.DataChannel;
25import org.webrtc.DefaultVideoDecoderFactory;
26import org.webrtc.DefaultVideoEncoderFactory;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.MediaConstraints;
30import org.webrtc.MediaStream;
31import org.webrtc.PeerConnection;
32import org.webrtc.PeerConnectionFactory;
33import org.webrtc.RtpReceiver;
34import org.webrtc.SdpObserver;
35import org.webrtc.SessionDescription;
36import org.webrtc.SurfaceTextureHelper;
37import org.webrtc.VideoSource;
38import org.webrtc.VideoTrack;
39
40import java.util.ArrayList;
41import java.util.Collections;
42import java.util.List;
43import java.util.Set;
44
45import javax.annotation.Nonnull;
46import javax.annotation.Nullable;
47
48import eu.siacs.conversations.Config;
49import eu.siacs.conversations.services.AppRTCAudioManager;
50
51public class WebRTCWrapper {
52
53 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
54
55 private static final int CAPTURING_RESOLUTION = 1920;
56 private static final int CAPTURING_MAX_FRAME_RATE = 30;
57
58 private final EventCallback eventCallback;
59 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
60 @Override
61 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
62 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
63 }
64 };
65 private final Handler mainHandler = new Handler(Looper.getMainLooper());
66 private VideoTrack localVideoTrack = null;
67 private VideoTrack remoteVideoTrack = null;
68 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
69 @Override
70 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
71 Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")");
72 //this is called after removeTrack or addTrack
73 //and should then trigger a content-add or content-remove or something
74 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
75 }
76
77 @Override
78 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
79 eventCallback.onConnectionChange(newState);
80 }
81
82 @Override
83 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
84
85 }
86
87 @Override
88 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
89 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
90 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
91 }
92
93 @Override
94 public void onIceConnectionReceivingChange(boolean b) {
95
96 }
97
98 @Override
99 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
100
101 }
102
103 @Override
104 public void onIceCandidate(IceCandidate iceCandidate) {
105 eventCallback.onIceCandidate(iceCandidate);
106 }
107
108 @Override
109 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
110
111 }
112
113 @Override
114 public void onAddStream(MediaStream mediaStream) {
115 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
116 if (videoTracks.size() > 0) {
117 remoteVideoTrack = videoTracks.get(0);
118 Log.d(Config.LOGTAG, "remote video track enabled?=" + remoteVideoTrack.enabled());
119 } else {
120 Log.d(Config.LOGTAG, "no remote video tracks found");
121 }
122 }
123
124 @Override
125 public void onRemoveStream(MediaStream mediaStream) {
126
127 }
128
129 @Override
130 public void onDataChannel(DataChannel dataChannel) {
131
132 }
133
134 @Override
135 public void onRenegotiationNeeded() {
136
137 }
138
139 @Override
140 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
141 Log.d(Config.LOGTAG, "onAddTrack()");
142
143 }
144 };
145 @Nullable
146 private PeerConnection peerConnection = null;
147 private AudioTrack localAudioTrack = null;
148 private AppRTCAudioManager appRTCAudioManager = null;
149 private Context context = null;
150 private EglBase eglBase = null;
151 private CapturerChoice capturerChoice;
152
153 public WebRTCWrapper(final EventCallback eventCallback) {
154 this.eventCallback = eventCallback;
155 }
156
157 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
158 PeerConnectionFactory.initialize(
159 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
160 );
161 this.eglBase = EglBase.create();
162 this.context = context;
163 mainHandler.post(() -> {
164 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
165 appRTCAudioManager.start(audioManagerEvents);
166 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
167 });
168 }
169
170 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
171 Preconditions.checkState(this.eglBase != null);
172 Preconditions.checkNotNull(media);
173 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
174 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
175 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
176 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
177 .createPeerConnectionFactory();
178
179
180 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
181
182 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
183
184 if (optionalCapturerChoice.isPresent()) {
185 this.capturerChoice = optionalCapturerChoice.get();
186 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
187 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
188 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
189 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
190 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
191 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
192
193 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
194
195 stream.addTrack(this.localVideoTrack);
196 }
197
198
199 if (media.contains(Media.AUDIO)) {
200 //set up audio track
201 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
202 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
203 stream.addTrack(this.localAudioTrack);
204 }
205
206
207 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
208 if (peerConnection == null) {
209 throw new InitializationException("Unable to create PeerConnection");
210 }
211 peerConnection.addStream(stream);
212 peerConnection.setAudioPlayout(true);
213 peerConnection.setAudioRecording(true);
214 this.peerConnection = peerConnection;
215 }
216
217 public void close() {
218 final PeerConnection peerConnection = this.peerConnection;
219 final CapturerChoice capturerChoice = this.capturerChoice;
220 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
221 final EglBase eglBase = this.eglBase;
222 if (peerConnection != null) {
223 peerConnection.dispose();
224 this.peerConnection = null;
225 }
226 if (audioManager != null) {
227 mainHandler.post(audioManager::stop);
228 }
229 this.localVideoTrack = null;
230 this.remoteVideoTrack = null;
231 if (capturerChoice != null) {
232 try {
233 capturerChoice.cameraVideoCapturer.stopCapture();
234 } catch (InterruptedException e) {
235 Log.e(Config.LOGTAG, "unable to stop capturing");
236 }
237 }
238 if (eglBase != null) {
239 eglBase.release();
240 this.eglBase = null;
241 }
242 }
243
244 void verifyClosed() {
245 if (this.peerConnection != null
246 || this.eglBase != null
247 || this.localVideoTrack != null
248 || this.remoteVideoTrack != null) {
249 throw new IllegalStateException("WebRTCWrapper hasn't been closed properly");
250 }
251 }
252
253 boolean isMicrophoneEnabled() {
254 final AudioTrack audioTrack = this.localAudioTrack;
255 if (audioTrack == null) {
256 throw new IllegalStateException("Local audio track does not exist (yet)");
257 }
258 return audioTrack.enabled();
259 }
260
261 void setMicrophoneEnabled(final boolean enabled) {
262 final AudioTrack audioTrack = this.localAudioTrack;
263 if (audioTrack == null) {
264 throw new IllegalStateException("Local audio track does not exist (yet)");
265 }
266 audioTrack.setEnabled(enabled);
267 }
268
269 public boolean isVideoEnabled() {
270 final VideoTrack videoTrack = this.localVideoTrack;
271 if (videoTrack == null) {
272 throw new IllegalStateException("Local video track does not exist");
273 }
274 return videoTrack.enabled();
275 }
276
277 public void setVideoEnabled(final boolean enabled) {
278 final VideoTrack videoTrack = this.localVideoTrack;
279 if (videoTrack == null) {
280 throw new IllegalStateException("Local video track does not exist");
281 }
282 videoTrack.setEnabled(enabled);
283 }
284
285 public ListenableFuture<SessionDescription> createOffer() {
286 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
287 final SettableFuture<SessionDescription> future = SettableFuture.create();
288 peerConnection.createOffer(new CreateSdpObserver() {
289 @Override
290 public void onCreateSuccess(SessionDescription sessionDescription) {
291 future.set(sessionDescription);
292 }
293
294 @Override
295 public void onCreateFailure(String s) {
296 Log.d(Config.LOGTAG, "create failure" + s);
297 future.setException(new IllegalStateException("Unable to create offer: " + s));
298 }
299 }, new MediaConstraints());
300 return future;
301 }, MoreExecutors.directExecutor());
302 }
303
304 public ListenableFuture<SessionDescription> createAnswer() {
305 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
306 final SettableFuture<SessionDescription> future = SettableFuture.create();
307 peerConnection.createAnswer(new CreateSdpObserver() {
308 @Override
309 public void onCreateSuccess(SessionDescription sessionDescription) {
310 future.set(sessionDescription);
311 }
312
313 @Override
314 public void onCreateFailure(String s) {
315 future.setException(new IllegalStateException("Unable to create answer: " + s));
316 }
317 }, new MediaConstraints());
318 return future;
319 }, MoreExecutors.directExecutor());
320 }
321
322 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
323 Log.d(EXTENDED_LOGGING_TAG, "setting local description:");
324 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
325 Log.d(EXTENDED_LOGGING_TAG, line);
326 }
327 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
328 final SettableFuture<Void> future = SettableFuture.create();
329 peerConnection.setLocalDescription(new SetSdpObserver() {
330 @Override
331 public void onSetSuccess() {
332 future.set(null);
333 }
334
335 @Override
336 public void onSetFailure(String s) {
337 Log.d(Config.LOGTAG, "unable to set local " + s);
338 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
339
340 }
341 }, sessionDescription);
342 return future;
343 }, MoreExecutors.directExecutor());
344 }
345
346 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
347 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
348 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
349 Log.d(EXTENDED_LOGGING_TAG, line);
350 }
351 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
352 final SettableFuture<Void> future = SettableFuture.create();
353 peerConnection.setRemoteDescription(new SetSdpObserver() {
354 @Override
355 public void onSetSuccess() {
356 future.set(null);
357 }
358
359 @Override
360 public void onSetFailure(String s) {
361 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
362
363 }
364 }, sessionDescription);
365 return future;
366 }, MoreExecutors.directExecutor());
367 }
368
369 @Nonnull
370 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
371 final PeerConnection peerConnection = this.peerConnection;
372 if (peerConnection == null) {
373 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
374 } else {
375 return Futures.immediateFuture(peerConnection);
376 }
377 }
378
379 public void addIceCandidate(IceCandidate iceCandidate) {
380 requirePeerConnection().addIceCandidate(iceCandidate);
381 }
382
383 private CameraEnumerator getCameraEnumerator() {
384 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
385 return new Camera2Enumerator(requireContext());
386 } else {
387 return new Camera1Enumerator();
388 }
389 }
390
391 private Optional<CapturerChoice> getVideoCapturer() {
392 final CameraEnumerator enumerator = getCameraEnumerator();
393 final String[] deviceNames = enumerator.getDeviceNames();
394 for (final String deviceName : deviceNames) {
395 if (enumerator.isFrontFacing(deviceName)) {
396 return Optional.fromNullable(of(enumerator, deviceName));
397 }
398 }
399 if (deviceNames.length == 0) {
400 return Optional.absent();
401 } else {
402 return Optional.fromNullable(of(enumerator, deviceNames[0]));
403 }
404 }
405
406 @Nullable
407 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName) {
408 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
409 if (capturer == null) {
410 return null;
411 }
412 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
413 Collections.sort(choices, (a, b) -> b.width - a.width);
414 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
415 if (captureFormat.width <= CAPTURING_RESOLUTION) {
416 return new CapturerChoice(capturer, captureFormat);
417 }
418 }
419 return null;
420 }
421
422 public PeerConnection.PeerConnectionState getState() {
423 return requirePeerConnection().connectionState();
424 }
425
426 EglBase.Context getEglBaseContext() {
427 return this.eglBase.getEglBaseContext();
428 }
429
430 public Optional<VideoTrack> getLocalVideoTrack() {
431 return Optional.fromNullable(this.localVideoTrack);
432 }
433
434 public Optional<VideoTrack> getRemoteVideoTrack() {
435 return Optional.fromNullable(this.remoteVideoTrack);
436 }
437
438 private PeerConnection requirePeerConnection() {
439 final PeerConnection peerConnection = this.peerConnection;
440 if (peerConnection == null) {
441 throw new IllegalStateException("initialize PeerConnection first");
442 }
443 return peerConnection;
444 }
445
446 private Context requireContext() {
447 final Context context = this.context;
448 if (context == null) {
449 throw new IllegalStateException("call setup first");
450 }
451 return context;
452 }
453
454 public AppRTCAudioManager getAudioManager() {
455 return appRTCAudioManager;
456 }
457
458 public interface EventCallback {
459 void onIceCandidate(IceCandidate iceCandidate);
460
461 void onConnectionChange(PeerConnection.PeerConnectionState newState);
462
463 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
464 }
465
466 private static abstract class SetSdpObserver implements SdpObserver {
467
468 @Override
469 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
470 throw new IllegalStateException("Not able to use SetSdpObserver");
471 }
472
473 @Override
474 public void onCreateFailure(String s) {
475 throw new IllegalStateException("Not able to use SetSdpObserver");
476 }
477
478 }
479
480 private static abstract class CreateSdpObserver implements SdpObserver {
481
482
483 @Override
484 public void onSetSuccess() {
485 throw new IllegalStateException("Not able to use CreateSdpObserver");
486 }
487
488
489 @Override
490 public void onSetFailure(String s) {
491 throw new IllegalStateException("Not able to use CreateSdpObserver");
492 }
493 }
494
495 public static class InitializationException extends Exception {
496
497 private InitializationException(String message) {
498 super(message);
499 }
500 }
501
502 private static class CapturerChoice {
503 private final CameraVideoCapturer cameraVideoCapturer;
504 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
505
506 public CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat) {
507 this.cameraVideoCapturer = cameraVideoCapturer;
508 this.captureFormat = captureFormat;
509 }
510
511 public int getFrameRate() {
512 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
513 }
514 }
515}