1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Preconditions;
8import com.google.common.base.Strings;
9import com.google.common.collect.Collections2;
10import com.google.common.collect.ImmutableList;
11import com.google.common.collect.ImmutableMap;
12import com.google.common.collect.Sets;
13import com.google.common.primitives.Ints;
14
15import org.webrtc.EglBase;
16import org.webrtc.IceCandidate;
17import org.webrtc.PeerConnection;
18import org.webrtc.VideoTrack;
19
20import java.util.ArrayDeque;
21import java.util.Arrays;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.List;
25import java.util.Map;
26import java.util.Set;
27
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.entities.Conversation;
30import eu.siacs.conversations.entities.Conversational;
31import eu.siacs.conversations.entities.Message;
32import eu.siacs.conversations.entities.RtpSessionStatus;
33import eu.siacs.conversations.services.AppRTCAudioManager;
34import eu.siacs.conversations.xml.Element;
35import eu.siacs.conversations.xml.Namespace;
36import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription;
37import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
38import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
39import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
40import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
41import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
42import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
43import eu.siacs.conversations.xmpp.stanzas.IqPacket;
44import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
45import rocks.xmpp.addr.Jid;
46
47public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
48
49 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
50 State.PROCEED,
51 State.SESSION_INITIALIZED,
52 State.SESSION_INITIALIZED_PRE_APPROVED,
53 State.SESSION_ACCEPTED
54 );
55
56 private static final List<State> TERMINATED = Arrays.asList(
57 State.TERMINATED_SUCCESS,
58 State.TERMINATED_DECLINED_OR_BUSY,
59 State.TERMINATED_CONNECTIVITY_ERROR,
60 State.TERMINATED_CANCEL_OR_TIMEOUT,
61 State.TERMINATED_APPLICATION_FAILURE
62 );
63
64 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
65
66 static {
67 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
68 transitionBuilder.put(State.NULL, ImmutableList.of(
69 State.PROPOSED,
70 State.SESSION_INITIALIZED,
71 State.TERMINATED_APPLICATION_FAILURE
72 ));
73 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
74 State.ACCEPTED,
75 State.PROCEED,
76 State.REJECTED,
77 State.RETRACTED,
78 State.TERMINATED_APPLICATION_FAILURE,
79 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
80 ));
81 transitionBuilder.put(State.PROCEED, ImmutableList.of(
82 State.SESSION_INITIALIZED_PRE_APPROVED,
83 State.TERMINATED_SUCCESS,
84 State.TERMINATED_APPLICATION_FAILURE,
85 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
86 ));
87 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
88 State.SESSION_ACCEPTED,
89 State.TERMINATED_SUCCESS,
90 State.TERMINATED_DECLINED_OR_BUSY,
91 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
92 State.TERMINATED_CANCEL_OR_TIMEOUT,
93 State.TERMINATED_APPLICATION_FAILURE
94 ));
95 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
96 State.SESSION_ACCEPTED,
97 State.TERMINATED_SUCCESS,
98 State.TERMINATED_DECLINED_OR_BUSY,
99 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
100 State.TERMINATED_CANCEL_OR_TIMEOUT,
101 State.TERMINATED_APPLICATION_FAILURE
102 ));
103 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
104 State.TERMINATED_SUCCESS,
105 State.TERMINATED_DECLINED_OR_BUSY,
106 State.TERMINATED_CONNECTIVITY_ERROR,
107 State.TERMINATED_CANCEL_OR_TIMEOUT,
108 State.TERMINATED_APPLICATION_FAILURE
109 ));
110 VALID_TRANSITIONS = transitionBuilder.build();
111 }
112
113 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
114 private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
115 private final Message message;
116 private State state = State.NULL;
117 private Set<Media> proposedMedia;
118 private RtpContentMap initiatorRtpContentMap;
119 private RtpContentMap responderRtpContentMap;
120 private long rtpConnectionStarted = 0; //time of 'connected'
121
122
123 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
124 super(jingleConnectionManager, id, initiator);
125 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
126 id.account,
127 id.with.asBareJid(),
128 false,
129 false
130 );
131 this.message = new Message(
132 conversation,
133 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
134 Message.TYPE_RTP_SESSION,
135 id.sessionId
136 );
137 }
138
139 private static State reasonToState(Reason reason) {
140 switch (reason) {
141 case SUCCESS:
142 return State.TERMINATED_SUCCESS;
143 case DECLINE:
144 case BUSY:
145 return State.TERMINATED_DECLINED_OR_BUSY;
146 case CANCEL:
147 case TIMEOUT:
148 return State.TERMINATED_CANCEL_OR_TIMEOUT;
149 case FAILED_APPLICATION:
150 return State.TERMINATED_APPLICATION_FAILURE;
151 default:
152 return State.TERMINATED_CONNECTIVITY_ERROR;
153 }
154 }
155
156 @Override
157 synchronized void deliverPacket(final JinglePacket jinglePacket) {
158 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
159 switch (jinglePacket.getAction()) {
160 case SESSION_INITIATE:
161 receiveSessionInitiate(jinglePacket);
162 break;
163 case TRANSPORT_INFO:
164 receiveTransportInfo(jinglePacket);
165 break;
166 case SESSION_ACCEPT:
167 receiveSessionAccept(jinglePacket);
168 break;
169 case SESSION_TERMINATE:
170 receiveSessionTerminate(jinglePacket);
171 break;
172 default:
173 respondOk(jinglePacket);
174 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
175 break;
176 }
177 }
178
179 @Override
180 synchronized void notifyRebound() {
181 if (TERMINATED.contains(this.state)) {
182 return;
183 }
184 webRTCWrapper.close();
185 if (!isInitiator() && isInState(State.PROPOSED,State.SESSION_INITIALIZED)) {
186 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
187 }
188 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
189 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
190 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
191 } else {
192 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
193 jingleConnectionManager.finishConnection(this);
194 }
195 }
196
197 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
198 respondOk(jinglePacket);
199 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
200 final State previous = this.state;
201 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
202 if (TERMINATED.contains(previous)) {
203 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
204 return;
205 }
206 webRTCWrapper.close();
207 final State target = reasonToState(wrapper.reason);
208 transitionOrThrow(target);
209 writeLogMessage(target);
210 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
211 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
212 }
213 jingleConnectionManager.finishConnection(this);
214 }
215
216 private void receiveTransportInfo(final JinglePacket jinglePacket) {
217 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
218 respondOk(jinglePacket);
219 final RtpContentMap contentMap;
220 try {
221 contentMap = RtpContentMap.of(jinglePacket);
222 } catch (IllegalArgumentException | NullPointerException e) {
223 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
224 return;
225 }
226 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
227 final Group originalGroup = rtpContentMap != null ? rtpContentMap.group : null;
228 final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
229 if (identificationTags.size() == 0) {
230 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
231 }
232 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
233 final String ufrag = content.getValue().transport.getAttribute("ufrag");
234 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
235 final String sdp = candidate.toSdpAttribute(ufrag);
236 final String sdpMid = content.getKey();
237 final int mLineIndex = identificationTags.indexOf(sdpMid);
238 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
239 if (isInState(State.SESSION_ACCEPTED)) {
240 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
241 this.webRTCWrapper.addIceCandidate(iceCandidate);
242 } else {
243 this.pendingIceCandidates.offer(iceCandidate);
244 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": put ICE candidate on backlog");
245 }
246 }
247 }
248 } else {
249 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
250 terminateWithOutOfOrder(jinglePacket);
251 }
252 }
253
254 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
255 if (isInitiator()) {
256 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
257 terminateWithOutOfOrder(jinglePacket);
258 return;
259 }
260 final RtpContentMap contentMap;
261 try {
262 contentMap = RtpContentMap.of(jinglePacket);
263 contentMap.requireContentDescriptions();
264 contentMap.requireDTLSFingerprint();
265 } catch (final IllegalArgumentException | IllegalStateException | NullPointerException e) {
266 respondOk(jinglePacket);
267 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
268 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
269 return;
270 }
271 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
272 final State target;
273 if (this.state == State.PROCEED) {
274 target = State.SESSION_INITIALIZED_PRE_APPROVED;
275 } else {
276 target = State.SESSION_INITIALIZED;
277 }
278 if (transition(target)) {
279 respondOk(jinglePacket);
280 this.initiatorRtpContentMap = contentMap;
281 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
282 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
283 sendSessionAccept();
284 } else {
285 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
286 startRinging();
287 }
288 } else {
289 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
290 terminateWithOutOfOrder(jinglePacket);
291 }
292 }
293
294 private void receiveSessionAccept(final JinglePacket jinglePacket) {
295 if (!isInitiator()) {
296 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
297 terminateWithOutOfOrder(jinglePacket);
298 return;
299 }
300 final RtpContentMap contentMap;
301 try {
302 contentMap = RtpContentMap.of(jinglePacket);
303 contentMap.requireContentDescriptions();
304 contentMap.requireDTLSFingerprint();
305 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
306 respondOk(jinglePacket);
307 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
308 webRTCWrapper.close();
309 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
310 return;
311 }
312 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
313 if (transition(State.SESSION_ACCEPTED)) {
314 respondOk(jinglePacket);
315 receiveSessionAccept(contentMap);
316 } else {
317 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
318 respondOk(jinglePacket);
319 }
320 }
321
322 private void receiveSessionAccept(final RtpContentMap contentMap) {
323 this.responderRtpContentMap = contentMap;
324 final SessionDescription sessionDescription;
325 try {
326 sessionDescription = SessionDescription.of(contentMap);
327 } catch (final IllegalArgumentException | NullPointerException e) {
328 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
329 webRTCWrapper.close();
330 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
331 return;
332 }
333 org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
334 org.webrtc.SessionDescription.Type.ANSWER,
335 sessionDescription.toString()
336 );
337 try {
338 this.webRTCWrapper.setRemoteDescription(answer).get();
339 } catch (Exception e) {
340 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", e);
341 webRTCWrapper.close();
342 sendSessionTerminate(Reason.FAILED_APPLICATION);
343 }
344 }
345
346 private void sendSessionAccept() {
347 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
348 if (rtpContentMap == null) {
349 throw new IllegalStateException("initiator RTP Content Map has not been set");
350 }
351 final SessionDescription offer;
352 try {
353 offer = SessionDescription.of(rtpContentMap);
354 } catch (final IllegalArgumentException | NullPointerException e) {
355 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
356 webRTCWrapper.close();
357 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
358 return;
359 }
360 sendSessionAccept(offer);
361 }
362
363 private void sendSessionAccept(final SessionDescription offer) {
364 discoverIceServers(iceServers -> sendSessionAccept(offer,iceServers));
365 }
366
367 private synchronized void sendSessionAccept(final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
368 if (TERMINATED.contains(this.state)) {
369 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
370 return;
371 }
372 try {
373 setupWebRTC(iceServers);
374 } catch (WebRTCWrapper.InitializationException e) {
375 sendSessionTerminate(Reason.FAILED_APPLICATION);
376 return;
377 }
378 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
379 org.webrtc.SessionDescription.Type.OFFER,
380 offer.toString()
381 );
382 try {
383 this.webRTCWrapper.setRemoteDescription(sdp).get();
384 addIceCandidatesFromBlackLog();
385 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
386 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
387 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
388 sendSessionAccept(respondingRtpContentMap);
389 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription);
390 } catch (Exception e) {
391 Log.d(Config.LOGTAG, "unable to send session accept", e);
392
393 }
394 }
395
396 private void addIceCandidatesFromBlackLog() {
397 while (!this.pendingIceCandidates.isEmpty()) {
398 final IceCandidate iceCandidate = this.pendingIceCandidates.poll();
399 this.webRTCWrapper.addIceCandidate(iceCandidate);
400 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added ICE candidate from back log " + iceCandidate);
401 }
402 }
403
404 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
405 this.responderRtpContentMap = rtpContentMap;
406 this.transitionOrThrow(State.SESSION_ACCEPTED);
407 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
408 Log.d(Config.LOGTAG, sessionAccept.toString());
409 send(sessionAccept);
410 }
411
412 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
413 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
414 switch (message.getName()) {
415 case "propose":
416 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
417 break;
418 case "proceed":
419 receiveProceed(from, serverMessageId, timestamp);
420 break;
421 case "retract":
422 receiveRetract(from, serverMessageId, timestamp);
423 break;
424 case "reject":
425 receiveReject(from, serverMessageId, timestamp);
426 break;
427 case "accept":
428 receiveAccept(from, serverMessageId, timestamp);
429 break;
430 default:
431 break;
432 }
433 }
434
435 void deliverFailedProceed() {
436 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
437 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
438 webRTCWrapper.close();
439 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
440 this.jingleConnectionManager.finishConnection(this);
441 }
442 }
443
444 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
445 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
446 if (originatedFromMyself) {
447 if (transition(State.ACCEPTED)) {
448 if (serverMsgId != null) {
449 this.message.setServerMsgId(serverMsgId);
450 }
451 this.message.setTime(timestamp);
452 this.message.setCarbon(true); //indicate that call was accepted on other device
453 this.writeLogMessageSuccess(0);
454 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
455 this.jingleConnectionManager.finishConnection(this);
456 } else {
457 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
458 }
459 } else {
460 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
461 }
462 }
463
464 private void receiveReject(Jid from, String serverMsgId, long timestamp) {
465 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
466 //reject from another one of my clients
467 if (originatedFromMyself) {
468 if (transition(State.REJECTED)) {
469 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
470 this.jingleConnectionManager.finishConnection(this);
471 if (serverMsgId != null) {
472 this.message.setServerMsgId(serverMsgId);
473 }
474 this.message.setTime(timestamp);
475 this.message.setCarbon(true); //indicate that call was rejected on other device
476 writeLogMessageMissed();
477 } else {
478 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
479 }
480 } else {
481 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
482 }
483 }
484
485 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
486 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
487 if (originatedFromMyself) {
488 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
489 } else if (transition(State.PROPOSED)) {
490 final Collection<RtpDescription> descriptions = Collections2.transform(
491 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
492 input -> (RtpDescription) input
493 );
494 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
495 Preconditions.checkState(!media.contains(Media.UNKNOWN),"RTP descriptions contain unknown media");
496 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received session proposal from "+from+" for "+media);
497 this.proposedMedia = Sets.newHashSet(media);
498 if (serverMsgId != null) {
499 this.message.setServerMsgId(serverMsgId);
500 }
501 this.message.setTime(timestamp);
502 startRinging();
503 } else {
504 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
505 }
506 }
507
508 private void startRinging() {
509 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
510 xmppConnectionService.getNotificationService().showIncomingCallNotification(id);
511 }
512
513 private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) {
514 if (from.equals(id.with)) {
515 if (isInitiator()) {
516 if (transition(State.PROCEED)) {
517 if (serverMsgId != null) {
518 this.message.setServerMsgId(serverMsgId);
519 }
520 this.message.setTime(timestamp);
521 this.sendSessionInitiate(State.SESSION_INITIALIZED_PRE_APPROVED);
522 } else {
523 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
524 }
525 } else {
526 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
527 }
528 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
529 if (transition(State.ACCEPTED)) {
530 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
531 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
532 this.jingleConnectionManager.finishConnection(this);
533 }
534 } else {
535 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
536 }
537 }
538
539 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
540 if (from.equals(id.with)) {
541 if (transition(State.RETRACTED)) {
542 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
543 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
544 if (serverMsgId != null) {
545 this.message.setServerMsgId(serverMsgId);
546 }
547 this.message.setTime(timestamp);
548 writeLogMessageMissed();
549 jingleConnectionManager.finishConnection(this);
550 } else {
551 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
552 }
553 } else {
554 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
555 }
556 }
557
558 private void sendSessionInitiate(final State targetState) {
559 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
560 discoverIceServers(iceServers -> sendSessionInitiate(targetState, iceServers));
561 }
562
563 private synchronized void sendSessionInitiate(final State targetState, final List<PeerConnection.IceServer> iceServers) {
564 if (TERMINATED.contains(this.state)) {
565 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
566 return;
567 }
568 try {
569 setupWebRTC(iceServers);
570 } catch (WebRTCWrapper.InitializationException e) {
571 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc");
572 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
573 return;
574 }
575 try {
576 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
577 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
578 Log.d(Config.LOGTAG, "description: " + webRTCSessionDescription.description);
579 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
580 sendSessionInitiate(rtpContentMap, targetState);
581 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
582 } catch (final Exception e) {
583 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", e);
584 webRTCWrapper.close();
585 if (isInState(targetState)) {
586 sendSessionTerminate(Reason.FAILED_APPLICATION);
587 } else {
588 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
589 }
590 }
591 }
592
593 private void sendSessionInitiate(RtpContentMap rtpContentMap, final State targetState) {
594 this.initiatorRtpContentMap = rtpContentMap;
595 this.transitionOrThrow(targetState);
596 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
597 send(sessionInitiate);
598 }
599
600 private void sendSessionTerminate(final Reason reason) {
601 sendSessionTerminate(reason, null);
602 }
603
604 private void sendSessionTerminate(final Reason reason, final String text) {
605 final State target = reasonToState(reason);
606 transitionOrThrow(target);
607 writeLogMessage(target);
608 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
609 jinglePacket.setReason(reason, text);
610 send(jinglePacket);
611 jingleConnectionManager.finishConnection(this);
612 }
613
614 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
615 final RtpContentMap transportInfo;
616 try {
617 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
618 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
619 } catch (Exception e) {
620 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
621 return;
622 }
623 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
624 send(jinglePacket);
625 }
626
627 private void send(final JinglePacket jinglePacket) {
628 jinglePacket.setTo(id.with);
629 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
630 if (response.getType() == IqPacket.TYPE.ERROR) {
631 final String errorCondition = response.getErrorCondition();
632 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
633 this.webRTCWrapper.close();
634 final State target;
635 if (Arrays.asList(
636 "service-unavailable",
637 "recipient-unavailable",
638 "remote-server-not-found",
639 "remote-server-timeout"
640 ).contains(errorCondition)) {
641 target = State.TERMINATED_CONNECTIVITY_ERROR;
642 } else {
643 target = State.TERMINATED_APPLICATION_FAILURE;
644 }
645 if (transition(target)) {
646 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminated session with " + id.with);
647 } else {
648 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not transitioning because already at state=" + this.state);
649 }
650
651 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
652 this.webRTCWrapper.close();
653 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
654 transition(State.TERMINATED_CONNECTIVITY_ERROR);
655 this.jingleConnectionManager.finishConnection(this);
656 }
657 });
658 }
659
660 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
661 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
662 webRTCWrapper.close();
663 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
664 respondWithOutOfOrder(jinglePacket);
665 jingleConnectionManager.finishConnection(this);
666 }
667
668 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
669 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
670 }
671
672 private void respondOk(final JinglePacket jinglePacket) {
673 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
674 }
675
676 public RtpEndUserState getEndUserState() {
677 switch (this.state) {
678 case PROPOSED:
679 case SESSION_INITIALIZED:
680 if (isInitiator()) {
681 return RtpEndUserState.RINGING;
682 } else {
683 return RtpEndUserState.INCOMING_CALL;
684 }
685 case PROCEED:
686 if (isInitiator()) {
687 return RtpEndUserState.RINGING;
688 } else {
689 return RtpEndUserState.ACCEPTING_CALL;
690 }
691 case SESSION_INITIALIZED_PRE_APPROVED:
692 if (isInitiator()) {
693 return RtpEndUserState.RINGING;
694 } else {
695 return RtpEndUserState.CONNECTING;
696 }
697 case SESSION_ACCEPTED:
698 final PeerConnection.PeerConnectionState state = webRTCWrapper.getState();
699 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
700 return RtpEndUserState.CONNECTED;
701 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
702 return RtpEndUserState.CONNECTING;
703 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
704 return RtpEndUserState.ENDING_CALL;
705 } else if (state == PeerConnection.PeerConnectionState.FAILED) {
706 return RtpEndUserState.CONNECTIVITY_ERROR;
707 } else {
708 return RtpEndUserState.ENDING_CALL;
709 }
710 case REJECTED:
711 case TERMINATED_DECLINED_OR_BUSY:
712 if (isInitiator()) {
713 return RtpEndUserState.DECLINED_OR_BUSY;
714 } else {
715 return RtpEndUserState.ENDED;
716 }
717 case TERMINATED_SUCCESS:
718 case ACCEPTED:
719 case RETRACTED:
720 case TERMINATED_CANCEL_OR_TIMEOUT:
721 return RtpEndUserState.ENDED;
722 case TERMINATED_CONNECTIVITY_ERROR:
723 return RtpEndUserState.CONNECTIVITY_ERROR;
724 case TERMINATED_APPLICATION_FAILURE:
725 return RtpEndUserState.APPLICATION_ERROR;
726 }
727 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
728 }
729
730
731 public synchronized void acceptCall() {
732 switch (this.state) {
733 case PROPOSED:
734 acceptCallFromProposed();
735 break;
736 case SESSION_INITIALIZED:
737 acceptCallFromSessionInitialized();
738 break;
739 default:
740 throw new IllegalStateException("Can not accept call from " + this.state);
741 }
742 }
743
744 public synchronized void rejectCall() {
745 switch (this.state) {
746 case PROPOSED:
747 rejectCallFromProposed();
748 break;
749 case SESSION_INITIALIZED:
750 rejectCallFromSessionInitiate();
751 break;
752 default:
753 throw new IllegalStateException("Can not reject call from " + this.state);
754 }
755 }
756
757 public synchronized void endCall() {
758 if (TERMINATED.contains(this.state)) {
759 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": received endCall() when session has already been terminated. nothing to do");
760 return;
761 }
762 if (isInState(State.PROPOSED) && !isInitiator()) {
763 rejectCallFromProposed();
764 return;
765 }
766 if (isInState(State.PROCEED)) {
767 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ending call while in state PROCEED just means ending the connection");
768 webRTCWrapper.close();
769 jingleConnectionManager.finishConnection(this);
770 transitionOrThrow(State.TERMINATED_SUCCESS); //arguably this wasn't success; but not a real failure either
771 return;
772 }
773 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
774 webRTCWrapper.close();
775 sendSessionTerminate(Reason.CANCEL);
776 return;
777 }
778 if (isInState(State.SESSION_INITIALIZED)) {
779 rejectCallFromSessionInitiate();
780 return;
781 }
782 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
783 webRTCWrapper.close();
784 sendSessionTerminate(Reason.SUCCESS);
785 return;
786 }
787 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
788 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
789 return;
790 }
791 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
792 }
793
794 private void setupWebRTC(final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
795 this.webRTCWrapper.setup(this.xmppConnectionService);
796 this.webRTCWrapper.initializePeerConnection(iceServers);
797 }
798
799 private void acceptCallFromProposed() {
800 transitionOrThrow(State.PROCEED);
801 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
802 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
803 this.sendJingleMessage("proceed");
804 }
805
806 private void rejectCallFromProposed() {
807 transitionOrThrow(State.REJECTED);
808 writeLogMessageMissed();
809 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
810 this.sendJingleMessage("reject");
811 jingleConnectionManager.finishConnection(this);
812 }
813
814 private void rejectCallFromSessionInitiate() {
815 webRTCWrapper.close();
816 sendSessionTerminate(Reason.DECLINE);
817 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
818 }
819
820 private void sendJingleMessage(final String action) {
821 sendJingleMessage(action, id.with);
822 }
823
824 private void sendJingleMessage(final String action, final Jid to) {
825 final MessagePacket messagePacket = new MessagePacket();
826 if ("proceed".equals(action)) {
827 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
828 }
829 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
830 messagePacket.setTo(to);
831 messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
832 messagePacket.addChild("store", "urn:xmpp:hints");
833 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
834 }
835
836 private void acceptCallFromSessionInitialized() {
837 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
838 sendSessionAccept();
839 }
840
841 private synchronized boolean isInState(State... state) {
842 return Arrays.asList(state).contains(this.state);
843 }
844
845 private synchronized boolean transition(final State target) {
846 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
847 if (validTransitions != null && validTransitions.contains(target)) {
848 this.state = target;
849 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
850 updateEndUserState();
851 updateOngoingCallNotification();
852 return true;
853 } else {
854 return false;
855 }
856 }
857
858 public void transitionOrThrow(final State target) {
859 if (!transition(target)) {
860 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
861 }
862 }
863
864 @Override
865 public void onIceCandidate(final IceCandidate iceCandidate) {
866 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
867 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
868 sendTransportInfo(iceCandidate.sdpMid, candidate);
869 }
870
871 @Override
872 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
873 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
874 updateEndUserState();
875 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
876 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
877 }
878 if (newState == PeerConnection.PeerConnectionState.FAILED) {
879 if (TERMINATED.contains(this.state)) {
880 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
881 return;
882 }
883 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
884 }
885 }
886
887 public AppRTCAudioManager getAudioManager() {
888 return webRTCWrapper.getAudioManager();
889 }
890
891 public boolean isMicrophoneEnabled() {
892 return webRTCWrapper.isMicrophoneEnabled();
893 }
894
895 public void setMicrophoneEnabled(final boolean enabled) {
896 webRTCWrapper.setMicrophoneEnabled(enabled);
897 }
898
899 @Override
900 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
901 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
902 }
903
904 private void updateEndUserState() {
905 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, getEndUserState());
906 }
907
908 private void updateOngoingCallNotification() {
909 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
910 xmppConnectionService.setOngoingCall(id);
911 } else {
912 xmppConnectionService.removeOngoingCall(id);
913 }
914 }
915
916 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
917 if (id.account.getXmppConnection().getFeatures().extendedServiceDiscovery()) {
918 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
919 request.setTo(Jid.of(id.account.getJid().getDomain()));
920 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
921 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
922 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
923 if (response.getType() == IqPacket.TYPE.RESULT) {
924 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
925 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
926 for (final Element child : children) {
927 if ("service".equals(child.getName())) {
928 final String type = child.getAttribute("type");
929 final String host = child.getAttribute("host");
930 final String sport = child.getAttribute("port");
931 final Integer port = sport == null ? null : Ints.tryParse(sport);
932 final String transport = child.getAttribute("transport");
933 final String username = child.getAttribute("username");
934 final String password = child.getAttribute("password");
935 if (Strings.isNullOrEmpty(host) || port == null) {
936 continue;
937 }
938 if (port < 0 || port > 65535) {
939 continue;
940 }
941 if (Arrays.asList("stun", "turn").contains(type) || Arrays.asList("udp", "tcp").contains(transport)) {
942 //TODO wrap ipv6 addresses
943 PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer.builder(String.format("%s:%s:%s?transport=%s", type, host, port, transport));
944 if (username != null && password != null) {
945 iceServerBuilder.setUsername(username);
946 iceServerBuilder.setPassword(password);
947 } else if (Arrays.asList("turn", "turns").contains(type)) {
948 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
949 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
950 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
951 continue;
952 }
953 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
954 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
955 listBuilder.add(iceServer);
956 }
957 }
958 }
959 }
960 List<PeerConnection.IceServer> iceServers = listBuilder.build();
961 if (iceServers.size() == 0) {
962 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
963 }
964 onIceServersDiscovered.onIceServersDiscovered(iceServers);
965 });
966 } else {
967 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
968 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
969 }
970 }
971
972 private void writeLogMessage(final State state) {
973 final long started = this.rtpConnectionStarted;
974 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
975 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
976 writeLogMessageSuccess(duration);
977 } else {
978 writeLogMessageMissed();
979 }
980 }
981
982 private void writeLogMessageSuccess(final long duration) {
983 this.message.setBody(new RtpSessionStatus(true, duration).toString());
984 this.writeMessage();
985 }
986
987 private void writeLogMessageMissed() {
988 this.message.setBody(new RtpSessionStatus(false, 0).toString());
989 this.writeMessage();
990 }
991
992 private void writeMessage() {
993 final Conversational conversational = message.getConversation();
994 if (conversational instanceof Conversation) {
995 ((Conversation) conversational).add(this.message);
996 xmppConnectionService.databaseBackend.createMessage(message);
997 xmppConnectionService.updateConversationUi();
998 } else {
999 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1000 }
1001 }
1002
1003 public State getState() {
1004 return this.state;
1005 }
1006
1007 public Optional<VideoTrack> geLocalVideoTrack() {
1008 return webRTCWrapper.getLocalVideoTrack();
1009 }
1010
1011 public Optional<VideoTrack> getRemoteVideoTrack() {
1012 return webRTCWrapper.getRemoteVideoTrack();
1013 }
1014
1015
1016 public EglBase.Context getEglBaseContext() {
1017 return webRTCWrapper.getEglBaseContext();
1018 }
1019
1020 public void setProposedMedia(final Set<Media> media) {
1021
1022 }
1023
1024 private interface OnIceServersDiscovered {
1025 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1026 }
1027}