1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Context;
4import android.os.Build;
5import android.os.Handler;
6import android.os.Looper;
7import android.util.Log;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.util.concurrent.Futures;
12import com.google.common.util.concurrent.ListenableFuture;
13import com.google.common.util.concurrent.MoreExecutors;
14import com.google.common.util.concurrent.SettableFuture;
15
16import org.webrtc.AudioSource;
17import org.webrtc.AudioTrack;
18import org.webrtc.Camera1Enumerator;
19import org.webrtc.Camera2Enumerator;
20import org.webrtc.CameraEnumerationAndroid;
21import org.webrtc.CameraEnumerator;
22import org.webrtc.CameraVideoCapturer;
23import org.webrtc.CandidatePairChangeEvent;
24import org.webrtc.DataChannel;
25import org.webrtc.DefaultVideoDecoderFactory;
26import org.webrtc.DefaultVideoEncoderFactory;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.MediaConstraints;
30import org.webrtc.MediaStream;
31import org.webrtc.MediaStreamTrack;
32import org.webrtc.PeerConnection;
33import org.webrtc.PeerConnectionFactory;
34import org.webrtc.RtpReceiver;
35import org.webrtc.RtpTransceiver;
36import org.webrtc.SdpObserver;
37import org.webrtc.SessionDescription;
38import org.webrtc.SurfaceTextureHelper;
39import org.webrtc.VideoSource;
40import org.webrtc.VideoTrack;
41
42import java.util.ArrayList;
43import java.util.Collections;
44import java.util.List;
45import java.util.Set;
46
47import javax.annotation.Nonnull;
48import javax.annotation.Nullable;
49
50import eu.siacs.conversations.Config;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52
53public class WebRTCWrapper {
54
55 private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName();
56
57 private static final int CAPTURING_RESOLUTION = 1920;
58 private static final int CAPTURING_MAX_FRAME_RATE = 30;
59
60 private final EventCallback eventCallback;
61 private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
62 @Override
63 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
64 eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices);
65 }
66 };
67 private final Handler mainHandler = new Handler(Looper.getMainLooper());
68 private VideoTrack localVideoTrack = null;
69 private VideoTrack remoteVideoTrack = null;
70 private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
71 @Override
72 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
73 Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")");
74 //this is called after removeTrack or addTrack
75 //and should then trigger a content-add or content-remove or something
76 //https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack
77 }
78
79 @Override
80 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
81 eventCallback.onConnectionChange(newState);
82 }
83
84 @Override
85 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
86
87 }
88
89 @Override
90 public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
91 Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote);
92 Log.d(Config.LOGTAG, "local candidate selected: " + event.local);
93 }
94
95 @Override
96 public void onIceConnectionReceivingChange(boolean b) {
97
98 }
99
100 @Override
101 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
102 Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
103 }
104
105 @Override
106 public void onIceCandidate(IceCandidate iceCandidate) {
107 eventCallback.onIceCandidate(iceCandidate);
108 }
109
110 @Override
111 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
112
113 }
114
115 @Override
116 public void onAddStream(MediaStream mediaStream) {
117 Log.d(EXTENDED_LOGGING_TAG, "onAddStream(numAudioTracks=" + mediaStream.audioTracks.size() + ",numVideoTracks=" + mediaStream.videoTracks.size() + ")");
118 final List<VideoTrack> videoTracks = mediaStream.videoTracks;
119 if (videoTracks.size() > 0) {
120 remoteVideoTrack = videoTracks.get(0);
121 Log.d(Config.LOGTAG, "remote video track enabled?=" + remoteVideoTrack.enabled());
122 } else {
123 Log.d(Config.LOGTAG, "no remote video tracks found");
124 }
125 }
126
127 @Override
128 public void onRemoveStream(MediaStream mediaStream) {
129
130 }
131
132 @Override
133 public void onDataChannel(DataChannel dataChannel) {
134
135 }
136
137 @Override
138 public void onRenegotiationNeeded() {
139
140 }
141
142 @Override
143 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
144 final MediaStreamTrack track = rtpReceiver.track();
145 Log.d(EXTENDED_LOGGING_TAG, "onAddTrack(kind=" + (track == null ? "null" : track.kind()) + ",numMediaStreams=" + mediaStreams.length + ")");
146 }
147
148 @Override
149 public void onTrack(RtpTransceiver transceiver) {
150 Log.d(EXTENDED_LOGGING_TAG, "onTrack(mid=" + transceiver.getMid() + ",media=" + transceiver.getMediaType() + ")");
151 }
152 };
153 @Nullable
154 private PeerConnection peerConnection = null;
155 private AudioTrack localAudioTrack = null;
156 private AppRTCAudioManager appRTCAudioManager = null;
157 private Context context = null;
158 private EglBase eglBase = null;
159 private CapturerChoice capturerChoice;
160
161 public WebRTCWrapper(final EventCallback eventCallback) {
162 this.eventCallback = eventCallback;
163 }
164
165 public void setup(final Context context, final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) {
166 PeerConnectionFactory.initialize(
167 PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
168 );
169 this.eglBase = EglBase.create();
170 this.context = context;
171 mainHandler.post(() -> {
172 appRTCAudioManager = AppRTCAudioManager.create(context, speakerPhonePreference);
173 appRTCAudioManager.start(audioManagerEvents);
174 eventCallback.onAudioDeviceChanged(appRTCAudioManager.getSelectedAudioDevice(), appRTCAudioManager.getAudioDevices());
175 });
176 }
177
178 public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
179 Preconditions.checkState(this.eglBase != null);
180 Preconditions.checkNotNull(media);
181 Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
182 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
183 .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
184 .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
185 .createPeerConnectionFactory();
186
187
188 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
189
190 final Optional<CapturerChoice> optionalCapturerChoice = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
191
192 if (optionalCapturerChoice.isPresent()) {
193 this.capturerChoice = optionalCapturerChoice.get();
194 final CameraVideoCapturer capturer = this.capturerChoice.cameraVideoCapturer;
195 final VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
196 SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("webrtc", eglBase.getEglBaseContext());
197 capturer.initialize(surfaceTextureHelper, requireContext(), videoSource.getCapturerObserver());
198 Log.d(Config.LOGTAG, String.format("start capturing at %dx%d@%d", capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate()));
199 capturer.startCapture(capturerChoice.captureFormat.width, capturerChoice.captureFormat.height, capturerChoice.getFrameRate());
200
201 this.localVideoTrack = peerConnectionFactory.createVideoTrack("my-video-track", videoSource);
202
203 stream.addTrack(this.localVideoTrack);
204 }
205
206
207 if (media.contains(Media.AUDIO)) {
208 //set up audio track
209 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
210 this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
211 stream.addTrack(this.localAudioTrack);
212 }
213
214
215 final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
216 rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp
217 final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver);
218 if (peerConnection == null) {
219 throw new InitializationException("Unable to create PeerConnection");
220 }
221 peerConnection.addStream(stream);
222 peerConnection.setAudioPlayout(true);
223 peerConnection.setAudioRecording(true);
224 this.peerConnection = peerConnection;
225 }
226
227 public void close() {
228 final PeerConnection peerConnection = this.peerConnection;
229 final CapturerChoice capturerChoice = this.capturerChoice;
230 final AppRTCAudioManager audioManager = this.appRTCAudioManager;
231 final EglBase eglBase = this.eglBase;
232 if (peerConnection != null) {
233 dispose(peerConnection);
234 this.peerConnection = null;
235 }
236 if (audioManager != null) {
237 mainHandler.post(audioManager::stop);
238 }
239 this.localVideoTrack = null;
240 this.remoteVideoTrack = null;
241 if (capturerChoice != null) {
242 try {
243 capturerChoice.cameraVideoCapturer.stopCapture();
244 } catch (InterruptedException e) {
245 Log.e(Config.LOGTAG, "unable to stop capturing");
246 }
247 }
248 if (eglBase != null) {
249 eglBase.release();
250 this.eglBase = null;
251 }
252 }
253
254 private static void dispose(final PeerConnection peerConnection) {
255 try {
256 peerConnection.dispose();
257 } catch (final IllegalStateException e) {
258 Log.e(Config.LOGTAG,"unable to dispose of peer connection", e);
259 }
260 }
261
262 void verifyClosed() {
263 if (this.peerConnection != null
264 || this.eglBase != null
265 || this.localVideoTrack != null
266 || this.remoteVideoTrack != null) {
267 final IllegalStateException e = new IllegalStateException("WebRTCWrapper hasn't been closed properly");
268 Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e);
269 throw e;
270 }
271 }
272
273 boolean isMicrophoneEnabled() {
274 final AudioTrack audioTrack = this.localAudioTrack;
275 if (audioTrack == null) {
276 throw new IllegalStateException("Local audio track does not exist (yet)");
277 }
278 return audioTrack.enabled();
279 }
280
281 void setMicrophoneEnabled(final boolean enabled) {
282 final AudioTrack audioTrack = this.localAudioTrack;
283 if (audioTrack == null) {
284 throw new IllegalStateException("Local audio track does not exist (yet)");
285 }
286 audioTrack.setEnabled(enabled);
287 }
288
289 public boolean isVideoEnabled() {
290 final VideoTrack videoTrack = this.localVideoTrack;
291 if (videoTrack == null) {
292 throw new IllegalStateException("Local video track does not exist");
293 }
294 return videoTrack.enabled();
295 }
296
297 public void setVideoEnabled(final boolean enabled) {
298 final VideoTrack videoTrack = this.localVideoTrack;
299 if (videoTrack == null) {
300 throw new IllegalStateException("Local video track does not exist");
301 }
302 videoTrack.setEnabled(enabled);
303 }
304
305 public ListenableFuture<SessionDescription> createOffer() {
306 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
307 final SettableFuture<SessionDescription> future = SettableFuture.create();
308 peerConnection.createOffer(new CreateSdpObserver() {
309 @Override
310 public void onCreateSuccess(SessionDescription sessionDescription) {
311 future.set(sessionDescription);
312 }
313
314 @Override
315 public void onCreateFailure(String s) {
316 Log.d(Config.LOGTAG, "create failure" + s);
317 future.setException(new IllegalStateException("Unable to create offer: " + s));
318 }
319 }, new MediaConstraints());
320 return future;
321 }, MoreExecutors.directExecutor());
322 }
323
324 public ListenableFuture<SessionDescription> createAnswer() {
325 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
326 final SettableFuture<SessionDescription> future = SettableFuture.create();
327 peerConnection.createAnswer(new CreateSdpObserver() {
328 @Override
329 public void onCreateSuccess(SessionDescription sessionDescription) {
330 future.set(sessionDescription);
331 }
332
333 @Override
334 public void onCreateFailure(String s) {
335 future.setException(new IllegalStateException("Unable to create answer: " + s));
336 }
337 }, new MediaConstraints());
338 return future;
339 }, MoreExecutors.directExecutor());
340 }
341
342 public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
343 Log.d(EXTENDED_LOGGING_TAG, "setting local description:");
344 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
345 Log.d(EXTENDED_LOGGING_TAG, line);
346 }
347 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
348 final SettableFuture<Void> future = SettableFuture.create();
349 peerConnection.setLocalDescription(new SetSdpObserver() {
350 @Override
351 public void onSetSuccess() {
352 future.set(null);
353 }
354
355 @Override
356 public void onSetFailure(String s) {
357 Log.d(Config.LOGTAG, "unable to set local " + s);
358 future.setException(new IllegalArgumentException("unable to set local session description: " + s));
359
360 }
361 }, sessionDescription);
362 return future;
363 }, MoreExecutors.directExecutor());
364 }
365
366 public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
367 Log.d(EXTENDED_LOGGING_TAG, "setting remote description:");
368 for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
369 Log.d(EXTENDED_LOGGING_TAG, line);
370 }
371 return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
372 final SettableFuture<Void> future = SettableFuture.create();
373 peerConnection.setRemoteDescription(new SetSdpObserver() {
374 @Override
375 public void onSetSuccess() {
376 future.set(null);
377 }
378
379 @Override
380 public void onSetFailure(String s) {
381 future.setException(new IllegalArgumentException("unable to set remote session description: " + s));
382
383 }
384 }, sessionDescription);
385 return future;
386 }, MoreExecutors.directExecutor());
387 }
388
389 @Nonnull
390 private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
391 final PeerConnection peerConnection = this.peerConnection;
392 if (peerConnection == null) {
393 return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
394 } else {
395 return Futures.immediateFuture(peerConnection);
396 }
397 }
398
399 public void addIceCandidate(IceCandidate iceCandidate) {
400 requirePeerConnection().addIceCandidate(iceCandidate);
401 }
402
403 private CameraEnumerator getCameraEnumerator() {
404 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
405 return new Camera2Enumerator(requireContext());
406 } else {
407 return new Camera1Enumerator();
408 }
409 }
410
411 private Optional<CapturerChoice> getVideoCapturer() {
412 final CameraEnumerator enumerator = getCameraEnumerator();
413 final String[] deviceNames = enumerator.getDeviceNames();
414 for (final String deviceName : deviceNames) {
415 if (enumerator.isFrontFacing(deviceName)) {
416 return Optional.fromNullable(of(enumerator, deviceName));
417 }
418 }
419 if (deviceNames.length == 0) {
420 return Optional.absent();
421 } else {
422 return Optional.fromNullable(of(enumerator, deviceNames[0]));
423 }
424 }
425
426 @Nullable
427 private static CapturerChoice of(CameraEnumerator enumerator, final String deviceName) {
428 final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null);
429 if (capturer == null) {
430 return null;
431 }
432 final ArrayList<CameraEnumerationAndroid.CaptureFormat> choices = new ArrayList<>(enumerator.getSupportedFormats(deviceName));
433 Collections.sort(choices, (a, b) -> b.width - a.width);
434 for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) {
435 if (captureFormat.width <= CAPTURING_RESOLUTION) {
436 return new CapturerChoice(capturer, captureFormat);
437 }
438 }
439 return null;
440 }
441
442 public PeerConnection.PeerConnectionState getState() {
443 return requirePeerConnection().connectionState();
444 }
445
446 EglBase.Context getEglBaseContext() {
447 return this.eglBase.getEglBaseContext();
448 }
449
450 public Optional<VideoTrack> getLocalVideoTrack() {
451 return Optional.fromNullable(this.localVideoTrack);
452 }
453
454 public Optional<VideoTrack> getRemoteVideoTrack() {
455 return Optional.fromNullable(this.remoteVideoTrack);
456 }
457
458 private PeerConnection requirePeerConnection() {
459 final PeerConnection peerConnection = this.peerConnection;
460 if (peerConnection == null) {
461 throw new IllegalStateException("initialize PeerConnection first");
462 }
463 return peerConnection;
464 }
465
466 private Context requireContext() {
467 final Context context = this.context;
468 if (context == null) {
469 throw new IllegalStateException("call setup first");
470 }
471 return context;
472 }
473
474 public AppRTCAudioManager getAudioManager() {
475 return appRTCAudioManager;
476 }
477
478 public interface EventCallback {
479 void onIceCandidate(IceCandidate iceCandidate);
480
481 void onConnectionChange(PeerConnection.PeerConnectionState newState);
482
483 void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
484 }
485
486 private static abstract class SetSdpObserver implements SdpObserver {
487
488 @Override
489 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
490 throw new IllegalStateException("Not able to use SetSdpObserver");
491 }
492
493 @Override
494 public void onCreateFailure(String s) {
495 throw new IllegalStateException("Not able to use SetSdpObserver");
496 }
497
498 }
499
500 private static abstract class CreateSdpObserver implements SdpObserver {
501
502
503 @Override
504 public void onSetSuccess() {
505 throw new IllegalStateException("Not able to use CreateSdpObserver");
506 }
507
508
509 @Override
510 public void onSetFailure(String s) {
511 throw new IllegalStateException("Not able to use CreateSdpObserver");
512 }
513 }
514
515 public static class InitializationException extends Exception {
516
517 private InitializationException(String message) {
518 super(message);
519 }
520 }
521
522 private static class CapturerChoice {
523 private final CameraVideoCapturer cameraVideoCapturer;
524 private final CameraEnumerationAndroid.CaptureFormat captureFormat;
525
526 public CapturerChoice(CameraVideoCapturer cameraVideoCapturer, CameraEnumerationAndroid.CaptureFormat captureFormat) {
527 this.cameraVideoCapturer = cameraVideoCapturer;
528 this.captureFormat = captureFormat;
529 }
530
531 public int getFrameRate() {
532 return Math.max(captureFormat.framerate.min, Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max));
533 }
534 }
535}