Daniel Gultsch created
src/main/java/eu/siacs/conversations/xml/Namespace.java | 1
src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java | 84
src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java | 19
src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java | 47
src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java | 11
5 files changed, 128 insertions(+), 34 deletions(-)
@@ -69,4 +69,5 @@ public final class Namespace {
public static final String UNIFIED_PUSH = "http://gultsch.de/xmpp/drafts/unified-push";
public static final String REPORTING = "urn:xmpp:reporting:1";
public static final String REPORTING_REASON_SPAM = "urn:xmpp:reporting:spam";
+ public static final String SDP_OFFER_ANSWER = "urn:ietf:rfc:3264";
}
@@ -635,7 +635,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
}
- private void resendCandidatesFromSdp(final SessionDescription answer) {
+ private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(final SessionDescription answer) {
final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
for(final SessionDescription.Media media : answer.media) {
final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
@@ -649,8 +649,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
}
}
- final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
- sendTransportInfo(candidates);
+ return candidateBuilder.build();
}
private void receiveContentReject(final JinglePacket jinglePacket) {
@@ -1406,8 +1405,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
+ final boolean includeCandidates = remoteHasSdpOfferAnswer();
try {
- setupWebRTC(media, iceServers);
+ setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close();
@@ -1421,8 +1421,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
this.webRTCWrapper.setRemoteDescription(sdp).get();
addIceCandidatesFromBlackLog();
org.webrtc.SessionDescription webRTCSessionDescription =
- this.webRTCWrapper.setLocalDescription().get();
- prepareSessionAccept(webRTCSessionDescription);
+ this.webRTCWrapper.setLocalDescription(includeCandidates).get();
+ prepareSessionAccept(webRTCSessionDescription, includeCandidates);
} catch (final Exception e) {
failureToAcceptSession(e);
}
@@ -1459,10 +1459,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private void prepareSessionAccept(
- final org.webrtc.SessionDescription webRTCSessionDescription) {
+ final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates) {
final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
+ final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
+ if (includeCandidates) {
+ candidates = parseCandidates(sessionDescription);
+ } else {
+ candidates = ImmutableMultimap.of();
+ }
this.responderRtpContentMap = respondingRtpContentMap;
storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
@@ -1472,8 +1478,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() {
@Override
public void onSuccess(final RtpContentMap outgoingContentMap) {
- sendSessionAccept(outgoingContentMap);
- webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ if (includeCandidates) {
+ Log.d(
+ Config.LOGTAG,
+ "including "
+ + candidates.size()
+ + " candidates in session accept");
+ sendSessionAccept(outgoingContentMap.withCandidates(candidates));
+ webRTCWrapper.resetPendingCandidates();
+ } else {
+ sendSessionAccept(outgoingContentMap);
+ webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ }
}
@Override
@@ -1871,8 +1887,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
+ final boolean includeCandidates = remoteHasSdpOfferAnswer();
try {
- setupWebRTC(media, iceServers);
+ setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close();
@@ -1881,8 +1898,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
try {
org.webrtc.SessionDescription webRTCSessionDescription =
- this.webRTCWrapper.setLocalDescription().get();
- prepareSessionInitiate(webRTCSessionDescription, targetState);
+ this.webRTCWrapper.setLocalDescription(includeCandidates).get();
+ prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
} catch (final Exception e) {
// TODO sending the error text is worthwhile as well. Especially for FailureToSet
// exceptions
@@ -1915,10 +1932,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private void prepareSessionInitiate(
- final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
+ final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates, final State targetState) {
final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
+ final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
+ if (includeCandidates) {
+ candidates = parseCandidates(sessionDescription);
+ } else {
+ candidates = ImmutableMultimap.of();
+ }
this.initiatorRtpContentMap = rtpContentMap;
final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
encryptSessionInitiate(rtpContentMap);
@@ -1927,8 +1950,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() {
@Override
public void onSuccess(final RtpContentMap outgoingContentMap) {
- sendSessionInitiate(outgoingContentMap, targetState);
- webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ if (includeCandidates) {
+ Log.d(
+ Config.LOGTAG,
+ "including "
+ + candidates.size()
+ + " candidates in session initiate");
+ sendSessionInitiate(outgoingContentMap.withCandidates(candidates), targetState);
+ webRTCWrapper.resetPendingCandidates();
+ } else {
+ sendSessionInitiate(outgoingContentMap, targetState);
+ webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ }
}
@Override
@@ -2031,11 +2064,6 @@ public class JingleRtpConnection extends AbstractJingleConnection
send(jinglePacket);
}
- private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
- // TODO send all candidates in one transport-info
- }
-
-
private void send(final JinglePacket jinglePacket) {
jinglePacket.setTo(id.with);
xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
@@ -2400,10 +2428,10 @@ public class JingleRtpConnection extends AbstractJingleConnection
finish();
}
- private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
+ private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers, final boolean trickle) throws WebRTCWrapper.InitializationException {
this.jingleConnectionManager.ensureConnectionIsRegistered(this);
this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
- this.webRTCWrapper.initializePeerConnection(media, iceServers);
+ this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
}
private void acceptCallFromProposed() {
@@ -2736,7 +2764,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
private SessionDescription setLocalSessionDescription()
throws ExecutionException, InterruptedException {
final org.webrtc.SessionDescription sessionDescription =
- this.webRTCWrapper.setLocalDescription().get();
+ this.webRTCWrapper.setLocalDescription(false).get();
return SessionDescription.parse(sessionDescription.description);
}
@@ -3024,6 +3052,14 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private boolean remoteHasVideoFeature() {
+ return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
+ }
+
+ private boolean remoteHasSdpOfferAnswer() {
+ return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
+ }
+
+ private boolean remoteHasFeature(final String feature) {
final Contact contact = id.getContact();
final Presence presence =
contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
@@ -3031,7 +3067,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
presence == null ? null : presence.getServiceDiscoveryResult();
final List<String> features =
serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
- return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
+ return features != null && features.contains(feature);
}
private interface OnIceServersDiscovered {
@@ -8,6 +8,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
@@ -196,6 +197,24 @@ public class RtpContentMap {
dt.senders, null, dt.transport.cloneWrapper())));
}
+ RtpContentMap withCandidates(
+ ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates) {
+ final ImmutableMap.Builder<String, DescriptionTransport> contentBuilder =
+ new ImmutableMap.Builder<>();
+ for (final Map.Entry<String, DescriptionTransport> entry : this.contents.entrySet()) {
+ final String name = entry.getKey();
+ final DescriptionTransport descriptionTransport = entry.getValue();
+ final var transport = descriptionTransport.transport;
+ contentBuilder.put(
+ name,
+ new DescriptionTransport(
+ descriptionTransport.senders,
+ descriptionTransport.description,
+ transport.withCandidates(candidates.get(name))));
+ }
+ return new RtpContentMap(group, contentBuilder.build());
+ }
+
public IceUdpTransportInfo.Credentials getDistinctCredentials() {
final Set<IceUdpTransportInfo.Credentials> allCredentials = getCredentials();
final IceUdpTransportInfo.Credentials credentials =
@@ -94,6 +94,8 @@ public class WebRTCWrapper {
private TrackWrapper<AudioTrack> localAudioTrack = null;
private TrackWrapper<VideoTrack> localVideoTrack = null;
private VideoTrack remoteVideoTrack = null;
+
+ private final SettableFuture<Void> iceGatheringComplete = SettableFuture.create();
private final PeerConnection.Observer peerConnectionObserver =
new PeerConnection.Observer() {
@Override
@@ -128,8 +130,11 @@ public class WebRTCWrapper {
@Override
public void onIceGatheringChange(
- PeerConnection.IceGatheringState iceGatheringState) {
+ final PeerConnection.IceGatheringState iceGatheringState) {
Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
+ if (iceGatheringState == PeerConnection.IceGatheringState.COMPLETE) {
+ iceGatheringComplete.set(null);
+ }
}
@Override
@@ -256,7 +261,9 @@ public class WebRTCWrapper {
}
synchronized void initializePeerConnection(
- final Set<Media> media, final List<PeerConnection.IceServer> iceServers)
+ final Set<Media> media,
+ final List<PeerConnection.IceServer> iceServers,
+ final boolean trickle)
throws InitializationException {
Preconditions.checkState(this.eglBase != null);
Preconditions.checkNotNull(media);
@@ -283,7 +290,7 @@ public class WebRTCWrapper {
.createAudioDeviceModule())
.createPeerConnectionFactory();
- final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers);
+ final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers, trickle);
final PeerConnection peerConnection =
requirePeerConnectionFactory()
.createPeerConnection(rtcConfig, peerConnectionObserver);
@@ -398,21 +405,27 @@ public class WebRTCWrapper {
}
private static PeerConnection.RTCConfiguration buildConfiguration(
- final List<PeerConnection.IceServer> iceServers) {
+ final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
final PeerConnection.RTCConfiguration rtcConfig =
new PeerConnection.RTCConfiguration(iceServers);
rtcConfig.tcpCandidatePolicy =
PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp
- rtcConfig.continualGatheringPolicy =
- PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
+ if (trickle) {
+ rtcConfig.continualGatheringPolicy =
+ PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
+ } else {
+ rtcConfig.continualGatheringPolicy =
+ PeerConnection.ContinualGatheringPolicy.GATHER_ONCE;
+ }
rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
rtcConfig.enableImplicitRollback = true;
return rtcConfig;
}
- void reconfigurePeerConnection(final List<PeerConnection.IceServer> iceServers) {
- requirePeerConnection().setConfiguration(buildConfiguration(iceServers));
+ void reconfigurePeerConnection(
+ final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
+ requirePeerConnection().setConfiguration(buildConfiguration(iceServers, trickle));
}
void restartIceAsync() {
@@ -443,6 +456,11 @@ public class WebRTCWrapper {
"setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is);
}
+ public void resetPendingCandidates() {
+ this.readyToReceivedIceCandidates.set(true);
+ this.iceCandidates.clear();
+ }
+
synchronized void close() {
final PeerConnection peerConnection = this.peerConnection;
final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;
@@ -561,7 +579,7 @@ public class WebRTCWrapper {
throw new IllegalStateException("Local video track does not exist");
}
- synchronized ListenableFuture<SessionDescription> setLocalDescription() {
+ synchronized ListenableFuture<SessionDescription> setLocalDescription(final boolean waitForCandidates) {
this.setIsReadyToReceiveIceCandidates(false);
return Futures.transformAsync(
getPeerConnectionFuture(),
@@ -575,7 +593,16 @@ public class WebRTCWrapper {
new SetSdpObserver() {
@Override
public void onSetSuccess() {
- future.setFuture(getLocalDescriptionFuture());
+ final var delay =
+ waitForCandidates
+ ? iceGatheringComplete
+ : Futures.immediateVoidFuture();
+ final var delayedSessionDescription =
+ Futures.transformAsync(
+ delay,
+ v -> getLocalDescriptionFuture(),
+ MoreExecutors.directExecutor());
+ future.setFuture(delayedSessionDescription);
}
@Override
@@ -12,6 +12,7 @@ import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
@@ -152,6 +153,16 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return transportInfo;
}
+ public IceUdpTransportInfo withCandidates(ImmutableCollection<Candidate> candidates) {
+ final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo();
+ transportInfo.setAttributes(new Hashtable<>(getAttributes()));
+ transportInfo.setChildren(this.getChildren());
+ for(final Candidate candidate : candidates) {
+ transportInfo.addChild(candidate);
+ }
+ return transportInfo;
+ }
+
public static class Credentials {
public final String ufrag;
public final String password;