1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.util.Log;
5
6import com.google.common.collect.ImmutableList;
7import com.google.common.collect.ImmutableMap;
8
9import org.webrtc.IceCandidate;
10import org.webrtc.PeerConnection;
11
12import java.util.ArrayDeque;
13import java.util.Arrays;
14import java.util.Collection;
15import java.util.Collections;
16import java.util.List;
17import java.util.Map;
18
19import eu.siacs.conversations.Config;
20import eu.siacs.conversations.ui.RtpSessionActivity;
21import eu.siacs.conversations.xml.Element;
22import eu.siacs.conversations.xml.Namespace;
23import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
24import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
25import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
26import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
27import rocks.xmpp.addr.Jid;
28
29public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
30
31 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
32
33 static {
34 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
35 transitionBuilder.put(State.NULL, ImmutableList.of(State.PROPOSED, State.SESSION_INITIALIZED));
36 transitionBuilder.put(State.PROPOSED, ImmutableList.of(State.ACCEPTED, State.PROCEED));
37 transitionBuilder.put(State.PROCEED, ImmutableList.of(State.SESSION_INITIALIZED));
38 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(State.SESSION_ACCEPTED));
39 VALID_TRANSITIONS = transitionBuilder.build();
40 }
41
42 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
43 private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
44 private State state = State.NULL;
45 private RtpContentMap initiatorRtpContentMap;
46 private RtpContentMap responderRtpContentMap;
47
48
49 public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
50 super(jingleConnectionManager, id, initiator);
51 }
52
53 @Override
54 void deliverPacket(final JinglePacket jinglePacket) {
55 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
56 switch (jinglePacket.getAction()) {
57 case SESSION_INITIATE:
58 receiveSessionInitiate(jinglePacket);
59 break;
60 case TRANSPORT_INFO:
61 receiveTransportInfo(jinglePacket);
62 break;
63 case SESSION_ACCEPT:
64 receiveSessionAccept(jinglePacket);
65 break;
66 default:
67 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
68 break;
69 }
70 }
71
72 private void receiveTransportInfo(final JinglePacket jinglePacket) {
73 if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) {
74 final RtpContentMap contentMap;
75 try {
76 contentMap = RtpContentMap.of(jinglePacket);
77 } catch (IllegalArgumentException | NullPointerException e) {
78 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
79 return;
80 }
81 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
82 final Group originalGroup = rtpContentMap != null ? rtpContentMap.group : null;
83 final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
84 if (identificationTags.size() == 0) {
85 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
86 }
87 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
88 final String ufrag = content.getValue().transport.getAttribute("ufrag");
89 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
90 final String sdp = candidate.toSdpAttribute(ufrag);
91 final String sdpMid = content.getKey();
92 final int mLineIndex = identificationTags.indexOf(sdpMid);
93 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
94 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
95 if (isInState(State.SESSION_ACCEPTED)) {
96 this.webRTCWrapper.addIceCandidate(iceCandidate);
97 } else {
98 this.pendingIceCandidates.push(iceCandidate);
99 }
100 }
101 }
102 } else {
103 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
104 }
105 }
106
107 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
108 if (isInitiator()) {
109 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
110 //TODO respond with out-of-order
111 return;
112 }
113 final RtpContentMap contentMap;
114 try {
115 contentMap = RtpContentMap.of(jinglePacket);
116 contentMap.requireContentDescriptions();
117 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
118 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
119 return;
120 }
121 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
122 final State oldState = this.state;
123 if (transition(State.SESSION_INITIALIZED)) {
124 this.initiatorRtpContentMap = contentMap;
125 if (oldState == State.PROCEED) {
126 Log.d(Config.LOGTAG, "automatically accepting");
127 sendSessionAccept();
128 } else {
129 Log.d(Config.LOGTAG, "start ringing");
130 //TODO start ringing
131 }
132 } else {
133 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
134 }
135 }
136
137 private void receiveSessionAccept(final JinglePacket jinglePacket) {
138 if (!isInitiator()) {
139 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
140 //TODO respond with out-of-order
141 return;
142 }
143 final RtpContentMap contentMap;
144 try {
145 contentMap = RtpContentMap.of(jinglePacket);
146 contentMap.requireContentDescriptions();
147 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
148 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
149 return;
150 }
151 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
152 if (transition(State.SESSION_ACCEPTED)) {
153 receiveSessionAccept(contentMap);
154 } else {
155 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
156 //TODO out-of-order
157 }
158 }
159
160 private void receiveSessionAccept(final RtpContentMap contentMap) {
161 this.responderRtpContentMap = contentMap;
162 org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
163 org.webrtc.SessionDescription.Type.ANSWER,
164 SessionDescription.of(contentMap).toString()
165 );
166 try {
167 this.webRTCWrapper.setRemoteDescription(answer).get();
168 } catch (Exception e) {
169 Log.d(Config.LOGTAG, "unable to receive session accept", e);
170 }
171 }
172
173 private void sendSessionAccept() {
174 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
175 if (rtpContentMap == null) {
176 throw new IllegalStateException("initiator RTP Content Map has not been set");
177 }
178 setupWebRTC();
179 final org.webrtc.SessionDescription offer = new org.webrtc.SessionDescription(
180 org.webrtc.SessionDescription.Type.OFFER,
181 SessionDescription.of(rtpContentMap).toString()
182 );
183 try {
184 this.webRTCWrapper.setRemoteDescription(offer).get();
185 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
186 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
187 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
188 sendSessionAccept(respondingRtpContentMap);
189 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription);
190 } catch (Exception e) {
191 Log.d(Config.LOGTAG, "unable to send session accept", e);
192
193 }
194 }
195
196 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
197 this.responderRtpContentMap = rtpContentMap;
198 this.transitionOrThrow(State.SESSION_ACCEPTED);
199 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
200 Log.d(Config.LOGTAG, sessionAccept.toString());
201 send(sessionAccept);
202 }
203
204 void deliveryMessage(final Jid from, final Element message) {
205 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
206 switch (message.getName()) {
207 case "propose":
208 receivePropose(from, message);
209 break;
210 case "proceed":
211 receiveProceed(from, message);
212 default:
213 break;
214 }
215 }
216
217 private void receivePropose(final Jid from, final Element propose) {
218 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
219 //TODO we can use initiator logic here
220 if (originatedFromMyself) {
221 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
222 } else if (transition(State.PROPOSED)) {
223 startRinging();
224 } else {
225 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
226 }
227 }
228
229 private void startRinging() {
230 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
231 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
232 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().asBareJid().toEscapedString());
233 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
234 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
235 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
236 xmppConnectionService.startActivity(intent);
237 }
238
239 private void receiveProceed(final Jid from, final Element proceed) {
240 if (from.equals(id.with)) {
241 if (isInitiator()) {
242 if (transition(State.PROCEED)) {
243 this.sendSessionInitiate();
244 } else {
245 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
246 }
247 } else {
248 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
249 }
250 } else {
251 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
252 }
253 }
254
255 private void sendSessionInitiate() {
256 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
257 setupWebRTC();
258 try {
259 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
260 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
261 Log.d(Config.LOGTAG, "description: " + webRTCSessionDescription.description);
262 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
263 sendSessionInitiate(rtpContentMap);
264 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
265 } catch (Exception e) {
266 Log.d(Config.LOGTAG, "unable to sendSessionInitiate", e);
267 }
268 }
269
270 private void sendSessionInitiate(RtpContentMap rtpContentMap) {
271 this.initiatorRtpContentMap = rtpContentMap;
272 this.transitionOrThrow(State.SESSION_INITIALIZED);
273 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
274 Log.d(Config.LOGTAG, sessionInitiate.toString());
275 send(sessionInitiate);
276 }
277
278 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
279 final RtpContentMap transportInfo;
280 try {
281 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
282 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
283 } catch (Exception e) {
284 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
285 return;
286 }
287 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
288 Log.d(Config.LOGTAG, jinglePacket.toString());
289 send(jinglePacket);
290 }
291
292 private void send(final JinglePacket jinglePacket) {
293 jinglePacket.setTo(id.with);
294 //TODO track errors
295 xmppConnectionService.sendIqPacket(id.account, jinglePacket, null);
296 }
297
298 public RtpEndUserState getEndUserState() {
299 switch (this.state) {
300 case PROPOSED:
301 if (isInitiator()) {
302 return RtpEndUserState.RINGING;
303 } else {
304 return RtpEndUserState.INCOMING_CALL;
305 }
306 case PROCEED:
307 if (isInitiator()) {
308 return RtpEndUserState.CONNECTING;
309 } else {
310 return RtpEndUserState.ACCEPTING_CALL;
311 }
312 case SESSION_INITIALIZED:
313 return RtpEndUserState.CONNECTING;
314 case SESSION_ACCEPTED:
315 final PeerConnection.PeerConnectionState state = webRTCWrapper.getState();
316 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
317 return RtpEndUserState.CONNECTED;
318 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
319 return RtpEndUserState.CONNECTING;
320 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
321 return RtpEndUserState.ENDING_CALL;
322 } else {
323 return RtpEndUserState.FAILED;
324 }
325 }
326 return RtpEndUserState.FAILED;
327 }
328
329
330 public void acceptCall() {
331 switch (this.state) {
332 case PROPOSED:
333 acceptCallFromProposed();
334 break;
335 case SESSION_INITIALIZED:
336 acceptCallFromSessionInitialized();
337 break;
338 default:
339 throw new IllegalStateException("Can not pick up call from " + this.state);
340 }
341 }
342
343 public void rejectCall() {
344 Log.d(Config.LOGTAG, "todo rejecting call");
345 }
346
347 public void endCall() {
348 if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) {
349 webRTCWrapper.close();
350 } else {
351 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": called 'endCall' while in state " + this.state);
352 }
353 }
354
355 private void setupWebRTC() {
356 this.webRTCWrapper.setup(this.xmppConnectionService);
357 this.webRTCWrapper.initializePeerConnection();
358 }
359
360 private void acceptCallFromProposed() {
361 transitionOrThrow(State.PROCEED);
362 final MessagePacket messagePacket = new MessagePacket();
363 messagePacket.setTo(id.with);
364 //Note that Movim needs 'accept', correct is 'proceed' https://github.com/movim/movim/issues/916
365 messagePacket.addChild("proceed", Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
366 Log.d(Config.LOGTAG, messagePacket.toString());
367 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
368 }
369
370 private void acceptCallFromSessionInitialized() {
371
372 }
373
374 private synchronized boolean isInState(State... state) {
375 return Arrays.asList(state).contains(this.state);
376 }
377
378 private synchronized boolean transition(final State target) {
379 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
380 if (validTransitions != null && validTransitions.contains(target)) {
381 this.state = target;
382 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
383 updateEndUserState();
384 return true;
385 } else {
386 return false;
387 }
388 }
389
390 public void transitionOrThrow(final State target) {
391 if (!transition(target)) {
392 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
393 }
394 }
395
396 @Override
397 public void onIceCandidate(final IceCandidate iceCandidate) {
398 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
399 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
400 sendTransportInfo(iceCandidate.sdpMid, candidate);
401 }
402
403 @Override
404 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
405 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": PeerConnectionState changed to "+newState);
406 updateEndUserState();
407 }
408
409 private void updateEndUserState() {
410 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, getEndUserState());
411 }
412}