1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Strings;
8import com.google.common.collect.ImmutableList;
9import com.google.common.collect.ImmutableMap;
10import com.google.common.primitives.Ints;
11
12import org.webrtc.EglBase;
13import org.webrtc.IceCandidate;
14import org.webrtc.PeerConnection;
15import org.webrtc.VideoTrack;
16
17import java.util.ArrayDeque;
18import java.util.Arrays;
19import java.util.Collection;
20import java.util.Collections;
21import java.util.List;
22import java.util.Map;
23import java.util.Set;
24
25import eu.siacs.conversations.Config;
26import eu.siacs.conversations.entities.Conversation;
27import eu.siacs.conversations.entities.Conversational;
28import eu.siacs.conversations.entities.Message;
29import eu.siacs.conversations.entities.RtpSessionStatus;
30import eu.siacs.conversations.services.AppRTCAudioManager;
31import eu.siacs.conversations.xml.Element;
32import eu.siacs.conversations.xml.Namespace;
33import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
34import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
35import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
36import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
37import eu.siacs.conversations.xmpp.stanzas.IqPacket;
38import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
39import rocks.xmpp.addr.Jid;
40
41public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
42
43 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
44 State.PROCEED,
45 State.SESSION_INITIALIZED,
46 State.SESSION_INITIALIZED_PRE_APPROVED,
47 State.SESSION_ACCEPTED
48 );
49
50 private static final List<State> TERMINATED = Arrays.asList(
51 State.TERMINATED_SUCCESS,
52 State.TERMINATED_DECLINED_OR_BUSY,
53 State.TERMINATED_CONNECTIVITY_ERROR,
54 State.TERMINATED_CANCEL_OR_TIMEOUT,
55 State.TERMINATED_APPLICATION_FAILURE
56 );
57
58 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
59
60 static {
61 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
62 transitionBuilder.put(State.NULL, ImmutableList.of(
63 State.PROPOSED,
64 State.SESSION_INITIALIZED,
65 State.TERMINATED_APPLICATION_FAILURE
66 ));
67 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
68 State.ACCEPTED,
69 State.PROCEED,
70 State.REJECTED,
71 State.RETRACTED,
72 State.TERMINATED_APPLICATION_FAILURE,
73 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
74 ));
75 transitionBuilder.put(State.PROCEED, ImmutableList.of(
76 State.SESSION_INITIALIZED_PRE_APPROVED,
77 State.TERMINATED_SUCCESS,
78 State.TERMINATED_APPLICATION_FAILURE,
79 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
80 ));
81 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
82 State.SESSION_ACCEPTED,
83 State.TERMINATED_SUCCESS,
84 State.TERMINATED_DECLINED_OR_BUSY,
85 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
86 State.TERMINATED_CANCEL_OR_TIMEOUT,
87 State.TERMINATED_APPLICATION_FAILURE
88 ));
89 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
90 State.SESSION_ACCEPTED,
91 State.TERMINATED_SUCCESS,
92 State.TERMINATED_DECLINED_OR_BUSY,
93 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
94 State.TERMINATED_CANCEL_OR_TIMEOUT,
95 State.TERMINATED_APPLICATION_FAILURE
96 ));
97 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
98 State.TERMINATED_SUCCESS,
99 State.TERMINATED_DECLINED_OR_BUSY,
100 State.TERMINATED_CONNECTIVITY_ERROR,
101 State.TERMINATED_CANCEL_OR_TIMEOUT,
102 State.TERMINATED_APPLICATION_FAILURE
103 ));
104 VALID_TRANSITIONS = transitionBuilder.build();
105 }
106
107 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
108 private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
109 private final Message message;
110 private State state = State.NULL;
111 private RtpContentMap initiatorRtpContentMap;
112 private RtpContentMap responderRtpContentMap;
113 private long rtpConnectionStarted = 0; //time of 'connected'
114
115
116 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
117 super(jingleConnectionManager, id, initiator);
118 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
119 id.account,
120 id.with.asBareJid(),
121 false,
122 false
123 );
124 this.message = new Message(
125 conversation,
126 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
127 Message.TYPE_RTP_SESSION,
128 id.sessionId
129 );
130 }
131
132 private static State reasonToState(Reason reason) {
133 switch (reason) {
134 case SUCCESS:
135 return State.TERMINATED_SUCCESS;
136 case DECLINE:
137 case BUSY:
138 return State.TERMINATED_DECLINED_OR_BUSY;
139 case CANCEL:
140 case TIMEOUT:
141 return State.TERMINATED_CANCEL_OR_TIMEOUT;
142 case FAILED_APPLICATION:
143 return State.TERMINATED_APPLICATION_FAILURE;
144 default:
145 return State.TERMINATED_CONNECTIVITY_ERROR;
146 }
147 }
148
149 @Override
150 synchronized void deliverPacket(final JinglePacket jinglePacket) {
151 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
152 switch (jinglePacket.getAction()) {
153 case SESSION_INITIATE:
154 receiveSessionInitiate(jinglePacket);
155 break;
156 case TRANSPORT_INFO:
157 receiveTransportInfo(jinglePacket);
158 break;
159 case SESSION_ACCEPT:
160 receiveSessionAccept(jinglePacket);
161 break;
162 case SESSION_TERMINATE:
163 receiveSessionTerminate(jinglePacket);
164 break;
165 default:
166 respondOk(jinglePacket);
167 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
168 break;
169 }
170 }
171
172 @Override
173 synchronized void notifyRebound() {
174 if (TERMINATED.contains(this.state)) {
175 return;
176 }
177 webRTCWrapper.close();
178 if (!isInitiator() && isInState(State.PROPOSED,State.SESSION_INITIALIZED)) {
179 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
180 }
181 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
182 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
183 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
184 } else {
185 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
186 jingleConnectionManager.finishConnection(this);
187 }
188 }
189
190 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
191 respondOk(jinglePacket);
192 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
193 final State previous = this.state;
194 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
195 if (TERMINATED.contains(previous)) {
196 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
197 return;
198 }
199 webRTCWrapper.close();
200 final State target = reasonToState(wrapper.reason);
201 transitionOrThrow(target);
202 writeLogMessage(target);
203 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
204 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
205 }
206 jingleConnectionManager.finishConnection(this);
207 }
208
209 private void receiveTransportInfo(final JinglePacket jinglePacket) {
210 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
211 respondOk(jinglePacket);
212 final RtpContentMap contentMap;
213 try {
214 contentMap = RtpContentMap.of(jinglePacket);
215 } catch (IllegalArgumentException | NullPointerException e) {
216 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
217 return;
218 }
219 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
220 final Group originalGroup = rtpContentMap != null ? rtpContentMap.group : null;
221 final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
222 if (identificationTags.size() == 0) {
223 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
224 }
225 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
226 final String ufrag = content.getValue().transport.getAttribute("ufrag");
227 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
228 final String sdp = candidate.toSdpAttribute(ufrag);
229 final String sdpMid = content.getKey();
230 final int mLineIndex = identificationTags.indexOf(sdpMid);
231 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
232 if (isInState(State.SESSION_ACCEPTED)) {
233 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
234 this.webRTCWrapper.addIceCandidate(iceCandidate);
235 } else {
236 this.pendingIceCandidates.offer(iceCandidate);
237 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": put ICE candidate on backlog");
238 }
239 }
240 }
241 } else {
242 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
243 terminateWithOutOfOrder(jinglePacket);
244 }
245 }
246
247 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
248 if (isInitiator()) {
249 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
250 terminateWithOutOfOrder(jinglePacket);
251 return;
252 }
253 final RtpContentMap contentMap;
254 try {
255 contentMap = RtpContentMap.of(jinglePacket);
256 contentMap.requireContentDescriptions();
257 contentMap.requireDTLSFingerprint();
258 } catch (final IllegalArgumentException | IllegalStateException | NullPointerException e) {
259 respondOk(jinglePacket);
260 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
261 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
262 return;
263 }
264 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
265 final State target;
266 if (this.state == State.PROCEED) {
267 target = State.SESSION_INITIALIZED_PRE_APPROVED;
268 } else {
269 target = State.SESSION_INITIALIZED;
270 }
271 if (transition(target)) {
272 respondOk(jinglePacket);
273 this.initiatorRtpContentMap = contentMap;
274 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
275 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
276 sendSessionAccept();
277 } else {
278 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
279 startRinging();
280 }
281 } else {
282 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
283 terminateWithOutOfOrder(jinglePacket);
284 }
285 }
286
287 private void receiveSessionAccept(final JinglePacket jinglePacket) {
288 if (!isInitiator()) {
289 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
290 terminateWithOutOfOrder(jinglePacket);
291 return;
292 }
293 final RtpContentMap contentMap;
294 try {
295 contentMap = RtpContentMap.of(jinglePacket);
296 contentMap.requireContentDescriptions();
297 contentMap.requireDTLSFingerprint();
298 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
299 respondOk(jinglePacket);
300 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
301 webRTCWrapper.close();
302 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
303 return;
304 }
305 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
306 if (transition(State.SESSION_ACCEPTED)) {
307 respondOk(jinglePacket);
308 receiveSessionAccept(contentMap);
309 } else {
310 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
311 respondOk(jinglePacket);
312 }
313 }
314
315 private void receiveSessionAccept(final RtpContentMap contentMap) {
316 this.responderRtpContentMap = contentMap;
317 final SessionDescription sessionDescription;
318 try {
319 sessionDescription = SessionDescription.of(contentMap);
320 } catch (final IllegalArgumentException | NullPointerException e) {
321 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
322 webRTCWrapper.close();
323 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
324 return;
325 }
326 org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
327 org.webrtc.SessionDescription.Type.ANSWER,
328 sessionDescription.toString()
329 );
330 try {
331 this.webRTCWrapper.setRemoteDescription(answer).get();
332 } catch (Exception e) {
333 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", e);
334 webRTCWrapper.close();
335 sendSessionTerminate(Reason.FAILED_APPLICATION);
336 }
337 }
338
339 private void sendSessionAccept() {
340 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
341 if (rtpContentMap == null) {
342 throw new IllegalStateException("initiator RTP Content Map has not been set");
343 }
344 final SessionDescription offer;
345 try {
346 offer = SessionDescription.of(rtpContentMap);
347 } catch (final IllegalArgumentException | NullPointerException e) {
348 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
349 webRTCWrapper.close();
350 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
351 return;
352 }
353 sendSessionAccept(offer);
354 }
355
356 private void sendSessionAccept(final SessionDescription offer) {
357 discoverIceServers(iceServers -> sendSessionAccept(offer,iceServers));
358 }
359
360 private synchronized void sendSessionAccept(final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
361 if (TERMINATED.contains(this.state)) {
362 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
363 return;
364 }
365 try {
366 setupWebRTC(iceServers);
367 } catch (WebRTCWrapper.InitializationException e) {
368 sendSessionTerminate(Reason.FAILED_APPLICATION);
369 return;
370 }
371 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
372 org.webrtc.SessionDescription.Type.OFFER,
373 offer.toString()
374 );
375 try {
376 this.webRTCWrapper.setRemoteDescription(sdp).get();
377 addIceCandidatesFromBlackLog();
378 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
379 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
380 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
381 sendSessionAccept(respondingRtpContentMap);
382 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription);
383 } catch (Exception e) {
384 Log.d(Config.LOGTAG, "unable to send session accept", e);
385
386 }
387 }
388
389 private void addIceCandidatesFromBlackLog() {
390 while (!this.pendingIceCandidates.isEmpty()) {
391 final IceCandidate iceCandidate = this.pendingIceCandidates.poll();
392 this.webRTCWrapper.addIceCandidate(iceCandidate);
393 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added ICE candidate from back log " + iceCandidate);
394 }
395 }
396
397 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
398 this.responderRtpContentMap = rtpContentMap;
399 this.transitionOrThrow(State.SESSION_ACCEPTED);
400 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
401 Log.d(Config.LOGTAG, sessionAccept.toString());
402 send(sessionAccept);
403 }
404
405 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
406 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
407 switch (message.getName()) {
408 case "propose":
409 receivePropose(from, serverMessageId, timestamp);
410 break;
411 case "proceed":
412 receiveProceed(from, serverMessageId, timestamp);
413 break;
414 case "retract":
415 receiveRetract(from, serverMessageId, timestamp);
416 break;
417 case "reject":
418 receiveReject(from, serverMessageId, timestamp);
419 break;
420 case "accept":
421 receiveAccept(from, serverMessageId, timestamp);
422 break;
423 default:
424 break;
425 }
426 }
427
428 void deliverFailedProceed() {
429 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
430 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
431 webRTCWrapper.close();
432 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
433 this.jingleConnectionManager.finishConnection(this);
434 }
435 }
436
437 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
438 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
439 if (originatedFromMyself) {
440 if (transition(State.ACCEPTED)) {
441 if (serverMsgId != null) {
442 this.message.setServerMsgId(serverMsgId);
443 }
444 this.message.setTime(timestamp);
445 this.message.setCarbon(true); //indicate that call was accepted on other device
446 this.writeLogMessageSuccess(0);
447 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
448 this.jingleConnectionManager.finishConnection(this);
449 } else {
450 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
451 }
452 } else {
453 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
454 }
455 }
456
457 private void receiveReject(Jid from, String serverMsgId, long timestamp) {
458 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
459 //reject from another one of my clients
460 if (originatedFromMyself) {
461 if (transition(State.REJECTED)) {
462 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
463 this.jingleConnectionManager.finishConnection(this);
464 if (serverMsgId != null) {
465 this.message.setServerMsgId(serverMsgId);
466 }
467 this.message.setTime(timestamp);
468 this.message.setCarbon(true); //indicate that call was rejected on other device
469 writeLogMessageMissed();
470 } else {
471 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
472 }
473 } else {
474 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
475 }
476 }
477
478 private void receivePropose(final Jid from, final String serverMsgId, final long timestamp) {
479 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
480 if (originatedFromMyself) {
481 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
482 } else if (transition(State.PROPOSED)) {
483 if (serverMsgId != null) {
484 this.message.setServerMsgId(serverMsgId);
485 }
486 this.message.setTime(timestamp);
487 startRinging();
488 } else {
489 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
490 }
491 }
492
493 private void startRinging() {
494 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
495 xmppConnectionService.getNotificationService().showIncomingCallNotification(id);
496 }
497
498 private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) {
499 if (from.equals(id.with)) {
500 if (isInitiator()) {
501 if (transition(State.PROCEED)) {
502 if (serverMsgId != null) {
503 this.message.setServerMsgId(serverMsgId);
504 }
505 this.message.setTime(timestamp);
506 this.sendSessionInitiate(State.SESSION_INITIALIZED_PRE_APPROVED);
507 } else {
508 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
509 }
510 } else {
511 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
512 }
513 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
514 if (transition(State.ACCEPTED)) {
515 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
516 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
517 this.jingleConnectionManager.finishConnection(this);
518 }
519 } else {
520 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
521 }
522 }
523
524 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
525 if (from.equals(id.with)) {
526 if (transition(State.RETRACTED)) {
527 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
528 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
529 if (serverMsgId != null) {
530 this.message.setServerMsgId(serverMsgId);
531 }
532 this.message.setTime(timestamp);
533 writeLogMessageMissed();
534 jingleConnectionManager.finishConnection(this);
535 } else {
536 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
537 }
538 } else {
539 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
540 }
541 }
542
543 private void sendSessionInitiate(final State targetState) {
544 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
545 discoverIceServers(iceServers -> sendSessionInitiate(targetState, iceServers));
546 }
547
548 private synchronized void sendSessionInitiate(final State targetState, final List<PeerConnection.IceServer> iceServers) {
549 if (TERMINATED.contains(this.state)) {
550 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
551 return;
552 }
553 try {
554 setupWebRTC(iceServers);
555 } catch (WebRTCWrapper.InitializationException e) {
556 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc");
557 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
558 return;
559 }
560 try {
561 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
562 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
563 Log.d(Config.LOGTAG, "description: " + webRTCSessionDescription.description);
564 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
565 sendSessionInitiate(rtpContentMap, targetState);
566 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
567 } catch (final Exception e) {
568 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", e);
569 webRTCWrapper.close();
570 if (isInState(targetState)) {
571 sendSessionTerminate(Reason.FAILED_APPLICATION);
572 } else {
573 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
574 }
575 }
576 }
577
578 private void sendSessionInitiate(RtpContentMap rtpContentMap, final State targetState) {
579 this.initiatorRtpContentMap = rtpContentMap;
580 this.transitionOrThrow(targetState);
581 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
582 send(sessionInitiate);
583 }
584
585 private void sendSessionTerminate(final Reason reason) {
586 sendSessionTerminate(reason, null);
587 }
588
589 private void sendSessionTerminate(final Reason reason, final String text) {
590 final State target = reasonToState(reason);
591 transitionOrThrow(target);
592 writeLogMessage(target);
593 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
594 jinglePacket.setReason(reason, text);
595 send(jinglePacket);
596 jingleConnectionManager.finishConnection(this);
597 }
598
599 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
600 final RtpContentMap transportInfo;
601 try {
602 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
603 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
604 } catch (Exception e) {
605 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
606 return;
607 }
608 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
609 send(jinglePacket);
610 }
611
612 private void send(final JinglePacket jinglePacket) {
613 jinglePacket.setTo(id.with);
614 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
615 if (response.getType() == IqPacket.TYPE.ERROR) {
616 final String errorCondition = response.getErrorCondition();
617 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
618 this.webRTCWrapper.close();
619 final State target;
620 if (Arrays.asList(
621 "service-unavailable",
622 "recipient-unavailable",
623 "remote-server-not-found",
624 "remote-server-timeout"
625 ).contains(errorCondition)) {
626 target = State.TERMINATED_CONNECTIVITY_ERROR;
627 } else {
628 target = State.TERMINATED_APPLICATION_FAILURE;
629 }
630 if (transition(target)) {
631 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminated session with " + id.with);
632 } else {
633 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not transitioning because already at state=" + this.state);
634 }
635
636 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
637 this.webRTCWrapper.close();
638 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
639 transition(State.TERMINATED_CONNECTIVITY_ERROR);
640 this.jingleConnectionManager.finishConnection(this);
641 }
642 });
643 }
644
645 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
646 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
647 webRTCWrapper.close();
648 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
649 respondWithOutOfOrder(jinglePacket);
650 jingleConnectionManager.finishConnection(this);
651 }
652
653 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
654 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
655 }
656
657 private void respondOk(final JinglePacket jinglePacket) {
658 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
659 }
660
661 public RtpEndUserState getEndUserState() {
662 switch (this.state) {
663 case PROPOSED:
664 case SESSION_INITIALIZED:
665 if (isInitiator()) {
666 return RtpEndUserState.RINGING;
667 } else {
668 return RtpEndUserState.INCOMING_CALL;
669 }
670 case PROCEED:
671 if (isInitiator()) {
672 return RtpEndUserState.RINGING;
673 } else {
674 return RtpEndUserState.ACCEPTING_CALL;
675 }
676 case SESSION_INITIALIZED_PRE_APPROVED:
677 if (isInitiator()) {
678 return RtpEndUserState.RINGING;
679 } else {
680 return RtpEndUserState.CONNECTING;
681 }
682 case SESSION_ACCEPTED:
683 final PeerConnection.PeerConnectionState state = webRTCWrapper.getState();
684 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
685 return RtpEndUserState.CONNECTED;
686 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
687 return RtpEndUserState.CONNECTING;
688 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
689 return RtpEndUserState.ENDING_CALL;
690 } else if (state == PeerConnection.PeerConnectionState.FAILED) {
691 return RtpEndUserState.CONNECTIVITY_ERROR;
692 } else {
693 return RtpEndUserState.ENDING_CALL;
694 }
695 case REJECTED:
696 case TERMINATED_DECLINED_OR_BUSY:
697 if (isInitiator()) {
698 return RtpEndUserState.DECLINED_OR_BUSY;
699 } else {
700 return RtpEndUserState.ENDED;
701 }
702 case TERMINATED_SUCCESS:
703 case ACCEPTED:
704 case RETRACTED:
705 case TERMINATED_CANCEL_OR_TIMEOUT:
706 return RtpEndUserState.ENDED;
707 case TERMINATED_CONNECTIVITY_ERROR:
708 return RtpEndUserState.CONNECTIVITY_ERROR;
709 case TERMINATED_APPLICATION_FAILURE:
710 return RtpEndUserState.APPLICATION_ERROR;
711 }
712 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
713 }
714
715
716 public synchronized void acceptCall() {
717 switch (this.state) {
718 case PROPOSED:
719 acceptCallFromProposed();
720 break;
721 case SESSION_INITIALIZED:
722 acceptCallFromSessionInitialized();
723 break;
724 default:
725 throw new IllegalStateException("Can not accept call from " + this.state);
726 }
727 }
728
729 public synchronized void rejectCall() {
730 switch (this.state) {
731 case PROPOSED:
732 rejectCallFromProposed();
733 break;
734 case SESSION_INITIALIZED:
735 rejectCallFromSessionInitiate();
736 break;
737 default:
738 throw new IllegalStateException("Can not reject call from " + this.state);
739 }
740 }
741
742 public synchronized void endCall() {
743 if (TERMINATED.contains(this.state)) {
744 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": received endCall() when session has already been terminated. nothing to do");
745 return;
746 }
747 if (isInState(State.PROPOSED) && !isInitiator()) {
748 rejectCallFromProposed();
749 return;
750 }
751 if (isInState(State.PROCEED)) {
752 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ending call while in state PROCEED just means ending the connection");
753 webRTCWrapper.close();
754 jingleConnectionManager.finishConnection(this);
755 transitionOrThrow(State.TERMINATED_SUCCESS); //arguably this wasn't success; but not a real failure either
756 return;
757 }
758 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
759 webRTCWrapper.close();
760 sendSessionTerminate(Reason.CANCEL);
761 return;
762 }
763 if (isInState(State.SESSION_INITIALIZED)) {
764 rejectCallFromSessionInitiate();
765 return;
766 }
767 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
768 webRTCWrapper.close();
769 sendSessionTerminate(Reason.SUCCESS);
770 return;
771 }
772 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
773 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
774 return;
775 }
776 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
777 }
778
779 private void setupWebRTC(final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
780 this.webRTCWrapper.setup(this.xmppConnectionService);
781 this.webRTCWrapper.initializePeerConnection(iceServers);
782 }
783
784 private void acceptCallFromProposed() {
785 transitionOrThrow(State.PROCEED);
786 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
787 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
788 this.sendJingleMessage("proceed");
789 }
790
791 private void rejectCallFromProposed() {
792 transitionOrThrow(State.REJECTED);
793 writeLogMessageMissed();
794 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
795 this.sendJingleMessage("reject");
796 jingleConnectionManager.finishConnection(this);
797 }
798
799 private void rejectCallFromSessionInitiate() {
800 webRTCWrapper.close();
801 sendSessionTerminate(Reason.DECLINE);
802 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
803 }
804
805 private void sendJingleMessage(final String action) {
806 sendJingleMessage(action, id.with);
807 }
808
809 private void sendJingleMessage(final String action, final Jid to) {
810 final MessagePacket messagePacket = new MessagePacket();
811 if ("proceed".equals(action)) {
812 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
813 }
814 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
815 messagePacket.setTo(to);
816 messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
817 messagePacket.addChild("store", "urn:xmpp:hints");
818 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
819 }
820
821 private void acceptCallFromSessionInitialized() {
822 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
823 sendSessionAccept();
824 }
825
826 private synchronized boolean isInState(State... state) {
827 return Arrays.asList(state).contains(this.state);
828 }
829
830 private synchronized boolean transition(final State target) {
831 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
832 if (validTransitions != null && validTransitions.contains(target)) {
833 this.state = target;
834 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
835 updateEndUserState();
836 updateOngoingCallNotification();
837 return true;
838 } else {
839 return false;
840 }
841 }
842
843 public void transitionOrThrow(final State target) {
844 if (!transition(target)) {
845 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
846 }
847 }
848
849 @Override
850 public void onIceCandidate(final IceCandidate iceCandidate) {
851 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
852 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
853 sendTransportInfo(iceCandidate.sdpMid, candidate);
854 }
855
856 @Override
857 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
858 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
859 updateEndUserState();
860 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
861 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
862 }
863 if (newState == PeerConnection.PeerConnectionState.FAILED) {
864 if (TERMINATED.contains(this.state)) {
865 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
866 return;
867 }
868 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
869 }
870 }
871
872 public AppRTCAudioManager getAudioManager() {
873 return webRTCWrapper.getAudioManager();
874 }
875
876 public boolean isMicrophoneEnabled() {
877 return webRTCWrapper.isMicrophoneEnabled();
878 }
879
880 public void setMicrophoneEnabled(final boolean enabled) {
881 webRTCWrapper.setMicrophoneEnabled(enabled);
882 }
883
884 @Override
885 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
886 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
887 }
888
889 private void updateEndUserState() {
890 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, getEndUserState());
891 }
892
893 private void updateOngoingCallNotification() {
894 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
895 xmppConnectionService.setOngoingCall(id);
896 } else {
897 xmppConnectionService.removeOngoingCall(id);
898 }
899 }
900
901 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
902 if (id.account.getXmppConnection().getFeatures().extendedServiceDiscovery()) {
903 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
904 request.setTo(Jid.of(id.account.getJid().getDomain()));
905 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
906 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
907 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
908 if (response.getType() == IqPacket.TYPE.RESULT) {
909 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
910 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
911 for (final Element child : children) {
912 if ("service".equals(child.getName())) {
913 final String type = child.getAttribute("type");
914 final String host = child.getAttribute("host");
915 final String sport = child.getAttribute("port");
916 final Integer port = sport == null ? null : Ints.tryParse(sport);
917 final String transport = child.getAttribute("transport");
918 final String username = child.getAttribute("username");
919 final String password = child.getAttribute("password");
920 if (Strings.isNullOrEmpty(host) || port == null) {
921 continue;
922 }
923 if (port < 0 || port > 65535) {
924 continue;
925 }
926 if (Arrays.asList("stun", "turn").contains(type) || Arrays.asList("udp", "tcp").contains(transport)) {
927 //TODO wrap ipv6 addresses
928 PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer.builder(String.format("%s:%s:%s?transport=%s", type, host, port, transport));
929 if (username != null && password != null) {
930 iceServerBuilder.setUsername(username);
931 iceServerBuilder.setPassword(password);
932 } else if (Arrays.asList("turn", "turns").contains(type)) {
933 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
934 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
935 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
936 continue;
937 }
938 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
939 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
940 listBuilder.add(iceServer);
941 }
942 }
943 }
944 }
945 List<PeerConnection.IceServer> iceServers = listBuilder.build();
946 if (iceServers.size() == 0) {
947 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
948 }
949 onIceServersDiscovered.onIceServersDiscovered(iceServers);
950 });
951 } else {
952 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
953 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
954 }
955 }
956
957 private void writeLogMessage(final State state) {
958 final long started = this.rtpConnectionStarted;
959 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
960 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
961 writeLogMessageSuccess(duration);
962 } else {
963 writeLogMessageMissed();
964 }
965 }
966
967 private void writeLogMessageSuccess(final long duration) {
968 this.message.setBody(new RtpSessionStatus(true, duration).toString());
969 this.writeMessage();
970 }
971
972 private void writeLogMessageMissed() {
973 this.message.setBody(new RtpSessionStatus(false, 0).toString());
974 this.writeMessage();
975 }
976
977 private void writeMessage() {
978 final Conversational conversational = message.getConversation();
979 if (conversational instanceof Conversation) {
980 ((Conversation) conversational).add(this.message);
981 xmppConnectionService.databaseBackend.createMessage(message);
982 xmppConnectionService.updateConversationUi();
983 } else {
984 throw new IllegalStateException("Somehow the conversation in a message was a stub");
985 }
986 }
987
988 public State getState() {
989 return this.state;
990 }
991
992 public Optional<VideoTrack> geLocalVideoTrack() {
993 return webRTCWrapper.getLocalVideoTrack();
994 }
995
996 public Optional<VideoTrack> getRemoteVideoTrack() {
997 return webRTCWrapper.getRemoteVideoTrack();
998 }
999
1000
1001 public EglBase.Context getEglBaseContext() {
1002 return webRTCWrapper.getEglBaseContext();
1003 }
1004
1005 private interface OnIceServersDiscovered {
1006 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1007 }
1008}