1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.util.Log;
5
6import com.google.common.collect.ImmutableList;
7import com.google.common.collect.ImmutableMap;
8
9import org.webrtc.IceCandidate;
10import org.webrtc.PeerConnection;
11
12import java.util.ArrayDeque;
13import java.util.Arrays;
14import java.util.Collection;
15import java.util.Collections;
16import java.util.List;
17import java.util.Map;
18
19import eu.siacs.conversations.Config;
20import eu.siacs.conversations.ui.RtpSessionActivity;
21import eu.siacs.conversations.xml.Element;
22import eu.siacs.conversations.xml.Namespace;
23import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
24import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
25import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
26import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
27import rocks.xmpp.addr.Jid;
28
29public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
30
31 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
32
33 static {
34 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
35 transitionBuilder.put(State.NULL, ImmutableList.of(State.PROPOSED, State.SESSION_INITIALIZED));
36 transitionBuilder.put(State.PROPOSED, ImmutableList.of(State.ACCEPTED, State.PROCEED));
37 transitionBuilder.put(State.PROCEED, ImmutableList.of(State.SESSION_INITIALIZED));
38 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(State.SESSION_ACCEPTED));
39 VALID_TRANSITIONS = transitionBuilder.build();
40 }
41
42 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
43 private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
44 private State state = State.NULL;
45 private RtpContentMap initiatorRtpContentMap;
46 private RtpContentMap responderRtpContentMap;
47
48
49 public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
50 super(jingleConnectionManager, id, initiator);
51 }
52
53 @Override
54 void deliverPacket(final JinglePacket jinglePacket) {
55 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
56 switch (jinglePacket.getAction()) {
57 case SESSION_INITIATE:
58 receiveSessionInitiate(jinglePacket);
59 break;
60 case TRANSPORT_INFO:
61 receiveTransportInfo(jinglePacket);
62 break;
63 case SESSION_ACCEPT:
64 receiveSessionAccept(jinglePacket);
65 break;
66 default:
67 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
68 break;
69 }
70 }
71
72 private void receiveTransportInfo(final JinglePacket jinglePacket) {
73 if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) {
74 final RtpContentMap contentMap;
75 try {
76 contentMap = RtpContentMap.of(jinglePacket);
77 } catch (IllegalArgumentException | NullPointerException e) {
78 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
79 return;
80 }
81 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
82 final Group originalGroup = rtpContentMap != null ? rtpContentMap.group : null;
83 final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
84 if (identificationTags.size() == 0) {
85 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
86 }
87 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
88 final String ufrag = content.getValue().transport.getAttribute("ufrag");
89 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
90 final String sdp = candidate.toSdpAttribute(ufrag);
91 final String sdpMid = content.getKey();
92 final int mLineIndex = identificationTags.indexOf(sdpMid);
93 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
94 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
95 if (isInState(State.SESSION_ACCEPTED)) {
96 this.webRTCWrapper.addIceCandidate(iceCandidate);
97 } else {
98 this.pendingIceCandidates.push(iceCandidate);
99 }
100 }
101 }
102 } else {
103 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
104 }
105 }
106
107 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
108 if (isInitiator()) {
109 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
110 //TODO respond with out-of-order
111 return;
112 }
113 final RtpContentMap contentMap;
114 try {
115 contentMap = RtpContentMap.of(jinglePacket);
116 contentMap.requireContentDescriptions();
117 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
118 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
119 return;
120 }
121 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
122 final State oldState = this.state;
123 if (transition(State.SESSION_INITIALIZED)) {
124 this.initiatorRtpContentMap = contentMap;
125 if (oldState == State.PROCEED) {
126 Log.d(Config.LOGTAG, "automatically accepting");
127 sendSessionAccept();
128 } else {
129 Log.d(Config.LOGTAG, "start ringing");
130 //TODO start ringing
131 }
132 } else {
133 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
134 }
135 }
136
137 private void receiveSessionAccept(final JinglePacket jinglePacket) {
138 if (!isInitiator()) {
139 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
140 //TODO respond with out-of-order
141 return;
142 }
143 final RtpContentMap contentMap;
144 try {
145 contentMap = RtpContentMap.of(jinglePacket);
146 contentMap.requireContentDescriptions();
147 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
148 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
149 return;
150 }
151 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
152 if (transition(State.SESSION_ACCEPTED)) {
153 receiveSessionAccept(contentMap);
154 } else {
155 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
156 //TODO out-of-order
157 }
158 }
159
160 private void receiveSessionAccept(final RtpContentMap contentMap) {
161 this.responderRtpContentMap = contentMap;
162 org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
163 org.webrtc.SessionDescription.Type.ANSWER,
164 SessionDescription.of(contentMap).toString()
165 );
166 try {
167 this.webRTCWrapper.setRemoteDescription(answer).get();
168 } catch (Exception e) {
169 Log.d(Config.LOGTAG, "unable to receive session accept", e);
170 }
171 }
172
173 private void sendSessionAccept() {
174 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
175 if (rtpContentMap == null) {
176 throw new IllegalStateException("initiator RTP Content Map has not been set");
177 }
178 setupWebRTC();
179 final org.webrtc.SessionDescription offer = new org.webrtc.SessionDescription(
180 org.webrtc.SessionDescription.Type.OFFER,
181 SessionDescription.of(rtpContentMap).toString()
182 );
183 try {
184 this.webRTCWrapper.setRemoteDescription(offer).get();
185 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
186 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
187 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
188 sendSessionAccept(respondingRtpContentMap);
189 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription);
190 } catch (Exception e) {
191 Log.d(Config.LOGTAG, "unable to send session accept", e);
192
193 }
194 }
195
196 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
197 this.responderRtpContentMap = rtpContentMap;
198 this.transitionOrThrow(State.SESSION_ACCEPTED);
199 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
200 Log.d(Config.LOGTAG, sessionAccept.toString());
201 send(sessionAccept);
202 }
203
204 void deliveryMessage(final Jid from, final Element message) {
205 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
206 switch (message.getName()) {
207 case "propose":
208 receivePropose(from, message);
209 break;
210 case "proceed":
211 receiveProceed(from, message);
212 default:
213 break;
214 }
215 }
216
217 private void receivePropose(final Jid from, final Element propose) {
218 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
219 //TODO we can use initiator logic here
220 if (originatedFromMyself) {
221 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
222 } else if (transition(State.PROPOSED)) {
223 startRinging();
224 } else {
225 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
226 }
227 }
228
229 private void startRinging() {
230 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
231 xmppConnectionService.getNotificationService().showIncomingCallNotification(id);
232 }
233
234 private void receiveProceed(final Jid from, final Element proceed) {
235 if (from.equals(id.with)) {
236 if (isInitiator()) {
237 if (transition(State.PROCEED)) {
238 this.sendSessionInitiate();
239 } else {
240 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
241 }
242 } else {
243 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
244 }
245 } else {
246 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
247 }
248 }
249
250 private void sendSessionInitiate() {
251 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
252 setupWebRTC();
253 try {
254 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
255 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
256 Log.d(Config.LOGTAG, "description: " + webRTCSessionDescription.description);
257 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
258 sendSessionInitiate(rtpContentMap);
259 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
260 } catch (Exception e) {
261 Log.d(Config.LOGTAG, "unable to sendSessionInitiate", e);
262 }
263 }
264
265 private void sendSessionInitiate(RtpContentMap rtpContentMap) {
266 this.initiatorRtpContentMap = rtpContentMap;
267 this.transitionOrThrow(State.SESSION_INITIALIZED);
268 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
269 Log.d(Config.LOGTAG, sessionInitiate.toString());
270 send(sessionInitiate);
271 }
272
273 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
274 final RtpContentMap transportInfo;
275 try {
276 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
277 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
278 } catch (Exception e) {
279 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
280 return;
281 }
282 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
283 Log.d(Config.LOGTAG, jinglePacket.toString());
284 send(jinglePacket);
285 }
286
287 private void send(final JinglePacket jinglePacket) {
288 jinglePacket.setTo(id.with);
289 //TODO track errors
290 xmppConnectionService.sendIqPacket(id.account, jinglePacket, null);
291 }
292
293 public RtpEndUserState getEndUserState() {
294 switch (this.state) {
295 case PROPOSED:
296 if (isInitiator()) {
297 return RtpEndUserState.RINGING;
298 } else {
299 return RtpEndUserState.INCOMING_CALL;
300 }
301 case PROCEED:
302 if (isInitiator()) {
303 return RtpEndUserState.CONNECTING;
304 } else {
305 return RtpEndUserState.ACCEPTING_CALL;
306 }
307 case SESSION_INITIALIZED:
308 return RtpEndUserState.CONNECTING;
309 case SESSION_ACCEPTED:
310 final PeerConnection.PeerConnectionState state = webRTCWrapper.getState();
311 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
312 return RtpEndUserState.CONNECTED;
313 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
314 return RtpEndUserState.CONNECTING;
315 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
316 return RtpEndUserState.ENDING_CALL;
317 } else {
318 return RtpEndUserState.FAILED;
319 }
320 }
321 return RtpEndUserState.FAILED;
322 }
323
324
325 public void acceptCall() {
326 switch (this.state) {
327 case PROPOSED:
328 acceptCallFromProposed();
329 break;
330 case SESSION_INITIALIZED:
331 acceptCallFromSessionInitialized();
332 break;
333 default:
334 throw new IllegalStateException("Can not pick up call from " + this.state);
335 }
336 }
337
338 public void rejectCall() {
339 Log.d(Config.LOGTAG, "todo rejecting call");
340 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
341 }
342
343 public void endCall() {
344 if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) {
345 webRTCWrapper.close();
346 } else {
347 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": called 'endCall' while in state " + this.state);
348 }
349 }
350
351 private void setupWebRTC() {
352 this.webRTCWrapper.setup(this.xmppConnectionService);
353 this.webRTCWrapper.initializePeerConnection();
354 }
355
356 private void acceptCallFromProposed() {
357 transitionOrThrow(State.PROCEED);
358 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
359 final MessagePacket messagePacket = new MessagePacket();
360 messagePacket.setTo(id.with);
361 //Note that Movim needs 'accept', correct is 'proceed' https://github.com/movim/movim/issues/916
362 messagePacket.addChild("proceed", Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
363 Log.d(Config.LOGTAG, messagePacket.toString());
364 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
365 }
366
367 private void acceptCallFromSessionInitialized() {
368 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
369 throw new IllegalStateException("accepting from this state has not been implemented yet");
370 }
371
372 private synchronized boolean isInState(State... state) {
373 return Arrays.asList(state).contains(this.state);
374 }
375
376 private synchronized boolean transition(final State target) {
377 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
378 if (validTransitions != null && validTransitions.contains(target)) {
379 this.state = target;
380 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
381 updateEndUserState();
382 return true;
383 } else {
384 return false;
385 }
386 }
387
388 public void transitionOrThrow(final State target) {
389 if (!transition(target)) {
390 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
391 }
392 }
393
394 @Override
395 public void onIceCandidate(final IceCandidate iceCandidate) {
396 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
397 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
398 sendTransportInfo(iceCandidate.sdpMid, candidate);
399 }
400
401 @Override
402 public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
403 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": PeerConnectionState changed to "+newState);
404 updateEndUserState();
405 }
406
407 private void updateEndUserState() {
408 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, getEndUserState());
409 }
410}