@@ -25,7 +25,6 @@ import org.webrtc.IceCandidate;
import org.webrtc.PeerConnection;
import org.webrtc.VideoTrack;
-import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -142,7 +141,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
- private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
+ //TODO convert to Queue<Map.Entry<String, Description>>?
+ private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
private final OmemoVerification omemoVerification = new OmemoVerification();
private final Message message;
private State state = State.NULL;
@@ -193,7 +193,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
synchronized void deliverPacket(final JinglePacket jinglePacket) {
- Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
switch (jinglePacket.getAction()) {
case SESSION_INITIATE:
receiveSessionInitiate(jinglePacket);
@@ -254,23 +253,29 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
private void receiveTransportInfo(final JinglePacket jinglePacket) {
//Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
- respondOk(jinglePacket);
final RtpContentMap contentMap;
try {
contentMap = RtpContentMap.of(jinglePacket);
- } catch (IllegalArgumentException | NullPointerException e) {
+ } catch (final IllegalArgumentException | NullPointerException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
+ respondOk(jinglePacket);
return;
}
final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
if (this.state == State.SESSION_ACCEPTED) {
+ //zero candidates + modified credentials are an ICE restart offer
+ if (checkForIceRestart(contentMap, jinglePacket)) {
+ return;
+ }
+ respondOk(jinglePacket);
try {
processCandidates(candidates);
} catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
}
} else {
- pendingIceCandidates.push(candidates);
+ respondOk(jinglePacket);
+ pendingIceCandidates.addAll(candidates);
}
} else {
if (isTerminated()) {
@@ -283,37 +288,106 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
}
- private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
- final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
- final Group originalGroup = rtpContentMap.group;
- final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
- if (identificationTags.size() == 0) {
- Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
+ private boolean checkForIceRestart(final RtpContentMap rtpContentMap, final JinglePacket jinglePacket) {
+ final RtpContentMap existing = getRemoteContentMap();
+ final Map<String, IceUdpTransportInfo.Credentials> existingCredentials = existing.getCredentials();
+ final Map<String, IceUdpTransportInfo.Credentials> newCredentials = rtpContentMap.getCredentials();
+ if (!existingCredentials.keySet().equals(newCredentials.keySet())) {
+ return false;
+ }
+ if (existingCredentials.equals(newCredentials)) {
+ return false;
+ }
+ final boolean isOffer = rtpContentMap.emptyCandidates();
+ Log.d(Config.LOGTAG, "detected ICE restart. offer=" + isOffer + " " + jinglePacket);
+ //TODO reset to 'actpass'?
+ final RtpContentMap restartContentMap = existing.modifiedCredentials(newCredentials);
+ try {
+ if (applyIceRestart(isOffer, restartContentMap)) {
+ return false;
+ } else {
+ Log.d(Config.LOGTAG, "responding with tie break");
+ //TODO respond with conflict
+ return true;
+ }
+ } catch (Exception e) {
+ Log.d(Config.LOGTAG, "failure to apply ICE restart. sending error", e);
+ //TODO send some kind of error
+ return true;
}
- processCandidates(identificationTags, contents);
}
- private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
+ private boolean applyIceRestart(final boolean isOffer, final RtpContentMap restartContentMap) throws ExecutionException, InterruptedException {
+ final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
+ final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
+ org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
+ if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
+ if (isInitiator()) {
+ //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
+ return false;
+ }
+ //rollback our own local description. should happen automatically but doesn't
+ webRTCWrapper.rollbackLocalDescription().get();
+ }
+ webRTCWrapper.setRemoteDescription(sdp).get();
+ if (isInitiator()) {
+ this.responderRtpContentMap = restartContentMap;
+ } else {
+ this.initiatorRtpContentMap = restartContentMap;
+ }
+ if (isOffer) {
+ webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
+ final SessionDescription localSessionDescription = setLocalSessionDescription();
+ setLocalContentMap(RtpContentMap.of(localSessionDescription));
+ webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ }
+ return true;
+ }
+
+ private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
- final String ufrag = content.getValue().transport.getAttribute("ufrag");
- for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
- final String sdp;
- try {
- sdp = candidate.toSdpAttribute(ufrag);
- } catch (IllegalArgumentException e) {
- Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
- continue;
- }
- final String sdpMid = content.getKey();
- final int mLineIndex = indices.indexOf(sdpMid);
- if (mLineIndex < 0) {
- Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
- }
- final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
- Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
- this.webRTCWrapper.addIceCandidate(iceCandidate);
+ processCandidate(content);
+ }
+ }
+
+ private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
+ final RtpContentMap rtpContentMap = getRemoteContentMap();
+ final List<String> indices = toIdentificationTags(rtpContentMap);
+ final String sdpMid = content.getKey(); //aka content name
+ final IceUdpTransportInfo transport = content.getValue().transport;
+ final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
+
+ //TODO check that credentials remained the same
+
+ for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
+ final String sdp;
+ try {
+ sdp = candidate.toSdpAttribute(credentials.ufrag);
+ } catch (final IllegalArgumentException e) {
+ Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
+ continue;
}
+ final int mLineIndex = indices.indexOf(sdpMid);
+ if (mLineIndex < 0) {
+ Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
+ }
+ final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
+ Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
+ this.webRTCWrapper.addIceCandidate(iceCandidate);
+ }
+ }
+
+ private RtpContentMap getRemoteContentMap() {
+ return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
+ }
+
+ private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
+ final Group originalGroup = rtpContentMap.group;
+ final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
+ if (identificationTags.size() == 0) {
+ Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
}
+ return identificationTags;
}
private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
@@ -401,11 +475,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
respondOk(jinglePacket);
-
- final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
- if (candidates.size() > 0) {
- pendingIceCandidates.push(candidates);
- }
+ pendingIceCandidates.addAll(contentMap.contents.entrySet());
if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
sendSessionAccept();
@@ -495,8 +565,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
sendSessionTerminate(Reason.FAILED_APPLICATION);
return;
}
- final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
- processCandidates(identificationTags, contentMap.contents.entrySet());
+ processCandidates(contentMap.contents.entrySet());
}
private void sendSessionAccept() {
@@ -558,9 +627,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
private void addIceCandidatesFromBlackLog() {
- while (!this.pendingIceCandidates.isEmpty()) {
- processCandidates(this.pendingIceCandidates.poll());
- Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
+ Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
+ while ((foo = this.pendingIceCandidates.poll()) != null) {
+ processCandidate(foo);
+ Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
}
}
@@ -1335,7 +1405,13 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
public void onIceCandidate(final IceCandidate iceCandidate) {
- final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
+ final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
+ final Collection<String> currentUfrags = Collections2.transform(rtpContentMap.getCredentials().values(), c -> c.ufrag);
+ final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, currentUfrags);
+ if (candidate == null) {
+ Log.d(Config.LOGTAG,"ignoring (not sending) candidate: "+iceCandidate.toString());
+ return;
+ }
Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
sendTransportInfo(iceCandidate.sdpMid, candidate);
}
@@ -1373,23 +1449,42 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
public void onRenegotiationNeeded() {
Log.d(Config.LOGTAG, "onRenegotiationNeeded()");
- this.webRTCWrapper.execute(this::renegotiate);
+ this.webRTCWrapper.execute(this::initiateIceRestart);
}
- private void renegotiate() {
+ private void initiateIceRestart() {
+ PeerConnection.SignalingState signalingState = webRTCWrapper.getSignalingState();
+ Log.d(Config.LOGTAG, "initiateIceRestart() - " + signalingState);
+ if (signalingState != PeerConnection.SignalingState.STABLE) {
+ return;
+ }
this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
+ final SessionDescription sessionDescription;
try {
- final SessionDescription sessionDescription = setLocalSessionDescription();
- final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
- setRenegotiatedContentMap(rtpContentMap);
- this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ sessionDescription = setLocalSessionDescription();
} catch (final Exception e) {
Log.d(Config.LOGTAG, "failed to renegotiate", e);
//TODO send some sort of failure (comparable to when initiating)
+ return;
}
+ final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
+ final RtpContentMap transportInfo = rtpContentMap.transportInfo();
+ final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
+ Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
+ jinglePacket.setTo(id.with);
+ xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
+ if (response.getType() == IqPacket.TYPE.RESULT) {
+ Log.d(Config.LOGTAG, "received success to our ice restart");
+ setLocalContentMap(rtpContentMap);
+ webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
+ } else {
+ Log.d(Config.LOGTAG, "received failure to our ice restart");
+ //TODO handle tie-break. Rollback?
+ }
+ });
}
- private void setRenegotiatedContentMap(final RtpContentMap rtpContentMap) {
+ private void setLocalContentMap(final RtpContentMap rtpContentMap) {
if (isInitiator()) {
this.initiatorRtpContentMap = rtpContentMap;
} else {
@@ -1,7 +1,5 @@
package eu.siacs.conversations.xmpp.jingle;
-import android.util.Log;
-
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -17,9 +15,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableDecl;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
-import eu.siacs.conversations.Config;
import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericTransportInfo;
@@ -137,7 +135,37 @@ public class RtpContentMap {
final IceUdpTransportInfo newTransportInfo = transportInfo.cloneWrapper();
newTransportInfo.addChild(candidate);
return new RtpContentMap(null, ImmutableMap.of(contentName, new DescriptionTransport(null, newTransportInfo)));
+ }
+
+ RtpContentMap transportInfo() {
+ return new RtpContentMap(
+ null,
+ Maps.transformValues(contents, dt -> new DescriptionTransport(null, dt.transport.cloneWrapper()))
+ );
+ }
+
+ public Map<String, IceUdpTransportInfo.Credentials> getCredentials() {
+ return Maps.transformValues(contents, dt -> dt.transport.getCredentials());
+ }
+ public boolean emptyCandidates() {
+ int count = 0;
+ for (DescriptionTransport descriptionTransport : contents.values()) {
+ count += descriptionTransport.transport.getCandidates().size();
+ }
+ return count == 0;
+ }
+
+ public RtpContentMap modifiedCredentials(Map<String, IceUdpTransportInfo.Credentials> credentialsMap) {
+ final ImmutableMap.Builder<String, DescriptionTransport> contentMapBuilder = new ImmutableMap.Builder<>();
+ for (final Map.Entry<String, DescriptionTransport> content : contents.entrySet()) {
+ final RtpDescription rtpDescription = content.getValue().description;
+ IceUdpTransportInfo transportInfo = content.getValue().transport;
+ final IceUdpTransportInfo.Credentials credentials = Objects.requireNonNull(credentialsMap.get(content.getKey()));
+ final IceUdpTransportInfo modifiedTransportInfo = transportInfo.modifyCredentials(credentials);
+ contentMapBuilder.put(content.getKey(), new DescriptionTransport(rtpDescription, modifiedTransportInfo));
+ }
+ return new RtpContentMap(this.group, contentMapBuilder.build());
}
public static class DescriptionTransport {
@@ -17,7 +17,6 @@ import com.google.common.util.concurrent.SettableFuture;
import org.webrtc.AudioSource;
import org.webrtc.AudioTrack;
-import org.webrtc.Camera1Enumerator;
import org.webrtc.Camera2Enumerator;
import org.webrtc.CameraEnumerationAndroid;
import org.webrtc.CameraEnumerator;
@@ -87,6 +86,7 @@ public class WebRTCWrapper {
private final EventCallback eventCallback;
private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false);
+ private final AtomicBoolean ignoreOnRenegotiationNeeded = new AtomicBoolean(false);
private final Queue<IceCandidate> iceCandidates = new LinkedList<>();
private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
@Override
@@ -163,6 +163,10 @@ public class WebRTCWrapper {
@Override
public void onRenegotiationNeeded() {
+ if (ignoreOnRenegotiationNeeded.get()) {
+ Log.d(EXTENDED_LOGGING_TAG, "ignoring onRenegotiationNeeded()");
+ return;
+ }
Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()");
final PeerConnection.PeerConnectionState currentState = peerConnection == null ? null : peerConnection.connectionState();
if (currentState != null && currentState != PeerConnection.PeerConnectionState.NEW) {
@@ -307,12 +311,12 @@ public class WebRTCWrapper {
}
void restartIce() {
- executorService.execute(()-> requirePeerConnection().restartIce());
+ executorService.execute(() -> requirePeerConnection().restartIce());
}
public void setIsReadyToReceiveIceCandidates(final boolean ready) {
readyToReceivedIceCandidates.set(ready);
- while(ready && iceCandidates.peek() != null) {
+ while (ready && iceCandidates.peek() != null) {
eventCallback.onIceCandidate(iceCandidates.poll());
}
}
@@ -452,6 +456,26 @@ public class WebRTCWrapper {
}, MoreExecutors.directExecutor());
}
+ public ListenableFuture<Void> rollbackLocalDescription() {
+ final SettableFuture<Void> future = SettableFuture.create();
+ final SessionDescription rollback = new SessionDescription(SessionDescription.Type.ROLLBACK, "");
+ ignoreOnRenegotiationNeeded.set(true);
+ requirePeerConnection().setLocalDescription(new SetSdpObserver() {
+ @Override
+ public void onSetSuccess() {
+ future.set(null);
+ ignoreOnRenegotiationNeeded.set(false);
+ }
+
+ @Override
+ public void onSetFailure(final String message) {
+ future.setException(new FailureToSetDescriptionException(message));
+ }
+ }, rollback);
+ return future;
+ }
+
+
private static void logDescription(final SessionDescription sessionDescription) {
for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
Log.d(EXTENDED_LOGGING_TAG, line);
@@ -552,6 +576,10 @@ public class WebRTCWrapper {
executorService.execute(command);
}
+ public PeerConnection.SignalingState getSignalingState() {
+ return requirePeerConnection().signalingState();
+ }
+
public interface EventCallback {
void onIceCandidate(IceCandidate iceCandidate);
@@ -1,6 +1,7 @@
package eu.siacs.conversations.xmpp.jingle.stanzas;
import com.google.common.base.Joiner;
+import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
@@ -8,6 +9,7 @@ import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.LinkedHashMap;
@@ -58,6 +60,12 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return fingerprint == null ? null : Fingerprint.upgrade(fingerprint);
}
+ public Credentials getCredentials() {
+ final String ufrag = this.getAttribute("ufrag");
+ final String password = this.getAttribute("pwd");
+ return new Credentials(ufrag, password);
+ }
+
public List<Candidate> getCandidates() {
final ImmutableList.Builder<Candidate> builder = new ImmutableList.Builder<>();
for (final Element child : getChildren()) {
@@ -74,6 +82,37 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return transportInfo;
}
+ public IceUdpTransportInfo modifyCredentials(Credentials credentials) {
+ final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo();
+ transportInfo.setAttribute("ufrag", credentials.ufrag);
+ transportInfo.setAttribute("pwd", credentials.password);
+ transportInfo.setChildren(getChildren());
+ return transportInfo;
+ }
+
+ public static class Credentials {
+ public final String ufrag;
+ public final String password;
+
+ public Credentials(String ufrag, String password) {
+ this.ufrag = ufrag;
+ this.password = password;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Credentials that = (Credentials) o;
+ return Objects.equal(ufrag, that.ufrag) && Objects.equal(password, that.password);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(ufrag, password);
+ }
+ }
+
public static class Candidate extends Element {
private Candidate() {
@@ -89,7 +128,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
}
// https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1
- public static Candidate fromSdpAttribute(final String attribute) {
+ public static Candidate fromSdpAttribute(final String attribute, Collection<String> currentUfrags) {
final String[] pair = attribute.split(":", 2);
if (pair.length == 2 && "candidate".equals(pair[0])) {
final String[] segments = pair[1].split(" ");
@@ -105,6 +144,10 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
for (int i = 6; i < segments.length - 1; i = i + 2) {
additional.put(segments[i], segments[i + 1]);
}
+ final String ufrag = additional.get("ufrag");
+ if (ufrag != null && !currentUfrags.contains(ufrag)) {
+ return null;
+ }
final Candidate candidate = new Candidate();
candidate.setAttribute("component", component);
candidate.setAttribute("foundation", foundation);