1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Preconditions;
8import com.google.common.base.Strings;
9import com.google.common.base.Throwables;
10import com.google.common.collect.Collections2;
11import com.google.common.collect.ImmutableList;
12import com.google.common.collect.ImmutableMap;
13import com.google.common.collect.Sets;
14import com.google.common.primitives.Ints;
15import com.google.common.util.concurrent.ListenableFuture;
16
17import org.webrtc.EglBase;
18import org.webrtc.IceCandidate;
19import org.webrtc.PeerConnection;
20import org.webrtc.VideoTrack;
21
22import java.util.ArrayDeque;
23import java.util.Arrays;
24import java.util.Collection;
25import java.util.Collections;
26import java.util.List;
27import java.util.Map;
28import java.util.Set;
29import java.util.concurrent.ScheduledFuture;
30import java.util.concurrent.TimeUnit;
31
32import eu.siacs.conversations.Config;
33import eu.siacs.conversations.crypto.axolotl.AxolotlService;
34import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
35import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
36import eu.siacs.conversations.entities.Account;
37import eu.siacs.conversations.entities.Conversation;
38import eu.siacs.conversations.entities.Conversational;
39import eu.siacs.conversations.entities.Message;
40import eu.siacs.conversations.entities.RtpSessionStatus;
41import eu.siacs.conversations.services.AppRTCAudioManager;
42import eu.siacs.conversations.utils.IP;
43import eu.siacs.conversations.xml.Element;
44import eu.siacs.conversations.xml.Namespace;
45import eu.siacs.conversations.xmpp.Jid;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
57
58 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
59 State.PROCEED,
60 State.SESSION_INITIALIZED_PRE_APPROVED,
61 State.SESSION_ACCEPTED
62 );
63 private static final long BUSY_TIME_OUT = 30;
64 private static final List<State> TERMINATED = Arrays.asList(
65 State.ACCEPTED,
66 State.REJECTED,
67 State.REJECTED_RACED,
68 State.RETRACTED,
69 State.RETRACTED_RACED,
70 State.TERMINATED_SUCCESS,
71 State.TERMINATED_DECLINED_OR_BUSY,
72 State.TERMINATED_CONNECTIVITY_ERROR,
73 State.TERMINATED_CANCEL_OR_TIMEOUT,
74 State.TERMINATED_APPLICATION_FAILURE
75 );
76
77 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
78
79 static {
80 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
81 transitionBuilder.put(State.NULL, ImmutableList.of(
82 State.PROPOSED,
83 State.SESSION_INITIALIZED,
84 State.TERMINATED_APPLICATION_FAILURE
85 ));
86 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
87 State.ACCEPTED,
88 State.PROCEED,
89 State.REJECTED,
90 State.RETRACTED,
91 State.TERMINATED_APPLICATION_FAILURE,
92 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
93 ));
94 transitionBuilder.put(State.PROCEED, ImmutableList.of(
95 State.REJECTED_RACED,
96 State.RETRACTED_RACED,
97 State.SESSION_INITIALIZED_PRE_APPROVED,
98 State.TERMINATED_SUCCESS,
99 State.TERMINATED_APPLICATION_FAILURE,
100 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
101 ));
102 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
103 State.SESSION_ACCEPTED,
104 State.TERMINATED_SUCCESS,
105 State.TERMINATED_DECLINED_OR_BUSY,
106 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
107 State.TERMINATED_CANCEL_OR_TIMEOUT,
108 State.TERMINATED_APPLICATION_FAILURE
109 ));
110 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
111 State.SESSION_ACCEPTED,
112 State.TERMINATED_SUCCESS,
113 State.TERMINATED_DECLINED_OR_BUSY,
114 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
115 State.TERMINATED_CANCEL_OR_TIMEOUT,
116 State.TERMINATED_APPLICATION_FAILURE
117 ));
118 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
119 State.TERMINATED_SUCCESS,
120 State.TERMINATED_DECLINED_OR_BUSY,
121 State.TERMINATED_CONNECTIVITY_ERROR,
122 State.TERMINATED_CANCEL_OR_TIMEOUT,
123 State.TERMINATED_APPLICATION_FAILURE
124 ));
125 VALID_TRANSITIONS = transitionBuilder.build();
126 }
127
128 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
129 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
130 private final OmemoVerification omemoVerification = new OmemoVerification();
131 private final Message message;
132 private State state = State.NULL;
133 private StateTransitionException stateTransitionException;
134 private Set<Media> proposedMedia;
135 private RtpContentMap initiatorRtpContentMap;
136 private RtpContentMap responderRtpContentMap;
137 private long rtpConnectionStarted = 0; //time of 'connected'
138 private long rtpConnectionEnded = 0;
139 private ScheduledFuture<?> ringingTimeoutFuture;
140
141 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
142 super(jingleConnectionManager, id, initiator);
143 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
144 id.account,
145 id.with.asBareJid(),
146 false,
147 false
148 );
149 this.message = new Message(
150 conversation,
151 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
152 Message.TYPE_RTP_SESSION,
153 id.sessionId
154 );
155 }
156
157 private static State reasonToState(Reason reason) {
158 switch (reason) {
159 case SUCCESS:
160 return State.TERMINATED_SUCCESS;
161 case DECLINE:
162 case BUSY:
163 return State.TERMINATED_DECLINED_OR_BUSY;
164 case CANCEL:
165 case TIMEOUT:
166 return State.TERMINATED_CANCEL_OR_TIMEOUT;
167 case FAILED_APPLICATION:
168 case SECURITY_ERROR:
169 case UNSUPPORTED_TRANSPORTS:
170 case UNSUPPORTED_APPLICATIONS:
171 return State.TERMINATED_APPLICATION_FAILURE;
172 default:
173 return State.TERMINATED_CONNECTIVITY_ERROR;
174 }
175 }
176
177 @Override
178 synchronized void deliverPacket(final JinglePacket jinglePacket) {
179 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
180 switch (jinglePacket.getAction()) {
181 case SESSION_INITIATE:
182 receiveSessionInitiate(jinglePacket);
183 break;
184 case TRANSPORT_INFO:
185 receiveTransportInfo(jinglePacket);
186 break;
187 case SESSION_ACCEPT:
188 receiveSessionAccept(jinglePacket);
189 break;
190 case SESSION_TERMINATE:
191 receiveSessionTerminate(jinglePacket);
192 break;
193 default:
194 respondOk(jinglePacket);
195 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
196 break;
197 }
198 }
199
200 @Override
201 synchronized void notifyRebound() {
202 if (isTerminated()) {
203 return;
204 }
205 webRTCWrapper.close();
206 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
207 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
208 }
209 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
210 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
211 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
212 } else {
213 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
214 finish();
215 }
216 }
217
218 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
219 respondOk(jinglePacket);
220 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
221 final State previous = this.state;
222 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
223 if (TERMINATED.contains(previous)) {
224 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
225 return;
226 }
227 webRTCWrapper.close();
228 final State target = reasonToState(wrapper.reason);
229 transitionOrThrow(target);
230 writeLogMessage(target);
231 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
232 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
233 }
234 finish();
235 }
236
237 private void receiveTransportInfo(final JinglePacket jinglePacket) {
238 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
239 respondOk(jinglePacket);
240 final RtpContentMap contentMap;
241 try {
242 contentMap = RtpContentMap.of(jinglePacket);
243 } catch (IllegalArgumentException | NullPointerException e) {
244 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
245 return;
246 }
247 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
248 if (this.state == State.SESSION_ACCEPTED) {
249 try {
250 processCandidates(candidates);
251 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
252 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
253 }
254 } else {
255 pendingIceCandidates.push(candidates);
256 }
257 } else {
258 if (isTerminated()) {
259 respondOk(jinglePacket);
260 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
261 } else {
262 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
263 terminateWithOutOfOrder(jinglePacket);
264 }
265 }
266 }
267
268 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
269 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
270 final Group originalGroup = rtpContentMap.group;
271 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
272 if (identificationTags.size() == 0) {
273 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
274 }
275 processCandidates(identificationTags, contents);
276 }
277
278 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
279 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
280 final String ufrag = content.getValue().transport.getAttribute("ufrag");
281 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
282 final String sdp;
283 try {
284 sdp = candidate.toSdpAttribute(ufrag);
285 } catch (IllegalArgumentException e) {
286 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
287 continue;
288 }
289 final String sdpMid = content.getKey();
290 final int mLineIndex = indices.indexOf(sdpMid);
291 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
292 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
293 this.webRTCWrapper.addIceCandidate(iceCandidate);
294 }
295 }
296 }
297
298 private RtpContentMap receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
299 final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket);
300 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
301 final AxolotlService.OmemoVerifiedPayload<RtpContentMap> omemoVerifiedPayload;
302 try {
303 omemoVerifiedPayload = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
304 } catch (final CryptoFailedException e) {
305 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
306 }
307 this.omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
308 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received verifiable DTLS fingerprint via "+this.omemoVerification);
309 return omemoVerifiedPayload.getPayload();
310 } else if (expectVerification) {
311 throw new SecurityException("DTLS fingerprint was unexpectedly not verifiable");
312 } else {
313 return receivedContentMap;
314 }
315 }
316
317 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
318 if (isInitiator()) {
319 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
320 terminateWithOutOfOrder(jinglePacket);
321 return;
322 }
323 final RtpContentMap contentMap;
324 try {
325 contentMap = receiveRtpContentMap(jinglePacket, false);
326 contentMap.requireContentDescriptions();
327 contentMap.requireDTLSFingerprint();
328 } catch (final RuntimeException e) {
329 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
330 respondOk(jinglePacket);
331 sendSessionTerminate(Reason.of(e), e.getMessage());
332 return;
333 }
334 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
335 final State target;
336 if (this.state == State.PROCEED) {
337 Preconditions.checkState(
338 proposedMedia != null && proposedMedia.size() > 0,
339 "proposed media must be set when processing pre-approved session-initiate"
340 );
341 if (!this.proposedMedia.equals(contentMap.getMedia())) {
342 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
343 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
344 this.proposedMedia,
345 contentMap.getMedia()
346 ));
347 return;
348 }
349 target = State.SESSION_INITIALIZED_PRE_APPROVED;
350 } else {
351 target = State.SESSION_INITIALIZED;
352 }
353 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
354 respondOk(jinglePacket);
355 //TODO Do not push empty set
356 pendingIceCandidates.push(contentMap.contents.entrySet());
357 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
358 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
359 sendSessionAccept();
360 } else {
361 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
362 startRinging();
363 }
364 } else {
365 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
366 terminateWithOutOfOrder(jinglePacket);
367 }
368 }
369
370 private void receiveSessionAccept(final JinglePacket jinglePacket) {
371 if (!isInitiator()) {
372 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
373 terminateWithOutOfOrder(jinglePacket);
374 return;
375 }
376 final RtpContentMap contentMap;
377 try {
378 contentMap = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
379 contentMap.requireContentDescriptions();
380 contentMap.requireDTLSFingerprint();
381 } catch (final RuntimeException e) {
382 respondOk(jinglePacket);
383 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
384 webRTCWrapper.close();
385 sendSessionTerminate(Reason.of(e), e.getMessage());
386 return;
387 }
388 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
389 if (!initiatorMedia.equals(contentMap.getMedia())) {
390 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
391 "Your session-included included media %s but our session-initiate was %s",
392 this.proposedMedia,
393 contentMap.getMedia()
394 ));
395 return;
396 }
397 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
398 if (transition(State.SESSION_ACCEPTED)) {
399 respondOk(jinglePacket);
400 receiveSessionAccept(contentMap);
401 } else {
402 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
403 respondOk(jinglePacket);
404 }
405 }
406
407 private void receiveSessionAccept(final RtpContentMap contentMap) {
408 this.responderRtpContentMap = contentMap;
409 final SessionDescription sessionDescription;
410 try {
411 sessionDescription = SessionDescription.of(contentMap);
412 } catch (final IllegalArgumentException | NullPointerException e) {
413 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
414 webRTCWrapper.close();
415 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
416 return;
417 }
418 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
419 org.webrtc.SessionDescription.Type.ANSWER,
420 sessionDescription.toString()
421 );
422 try {
423 this.webRTCWrapper.setRemoteDescription(answer).get();
424 } catch (final Exception e) {
425 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
426 webRTCWrapper.close();
427 sendSessionTerminate(Reason.FAILED_APPLICATION);
428 return;
429 }
430 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
431 processCandidates(identificationTags, contentMap.contents.entrySet());
432 }
433
434 private void sendSessionAccept() {
435 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
436 if (rtpContentMap == null) {
437 throw new IllegalStateException("initiator RTP Content Map has not been set");
438 }
439 final SessionDescription offer;
440 try {
441 offer = SessionDescription.of(rtpContentMap);
442 } catch (final IllegalArgumentException | NullPointerException e) {
443 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
444 webRTCWrapper.close();
445 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
446 return;
447 }
448 sendSessionAccept(rtpContentMap.getMedia(), offer);
449 }
450
451 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
452 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
453 }
454
455 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
456 if (isTerminated()) {
457 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
458 return;
459 }
460 try {
461 setupWebRTC(media, iceServers);
462 } catch (final WebRTCWrapper.InitializationException e) {
463 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
464 webRTCWrapper.close();
465 sendSessionTerminate(Reason.FAILED_APPLICATION);
466 return;
467 }
468 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
469 org.webrtc.SessionDescription.Type.OFFER,
470 offer.toString()
471 );
472 try {
473 this.webRTCWrapper.setRemoteDescription(sdp).get();
474 addIceCandidatesFromBlackLog();
475 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
476 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
477 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
478 sendSessionAccept(respondingRtpContentMap);
479 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
480 } catch (final Exception e) {
481 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(e));
482 webRTCWrapper.close();
483 sendSessionTerminate(Reason.FAILED_APPLICATION);
484 }
485 }
486
487 private void addIceCandidatesFromBlackLog() {
488 while (!this.pendingIceCandidates.isEmpty()) {
489 processCandidates(this.pendingIceCandidates.poll());
490 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
491 }
492 }
493
494 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
495 this.responderRtpContentMap = rtpContentMap;
496 this.transitionOrThrow(State.SESSION_ACCEPTED);
497 final RtpContentMap outgoingContentMap;
498 if (this.omemoVerification.hasDeviceId()) {
499 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
500 try {
501 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
502 outgoingContentMap = verifiedPayload.getPayload();
503 this.omemoVerification.setOrEnsureEqual(verifiedPayload);
504 } catch (final Exception e) {
505 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
506 }
507 } else {
508 outgoingContentMap = rtpContentMap;
509 }
510 final JinglePacket sessionAccept = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
511 send(sessionAccept);
512 }
513
514 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
515 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
516 switch (message.getName()) {
517 case "propose":
518 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
519 break;
520 case "proceed":
521 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
522 break;
523 case "retract":
524 receiveRetract(from, serverMessageId, timestamp);
525 break;
526 case "reject":
527 receiveReject(from, serverMessageId, timestamp);
528 break;
529 case "accept":
530 receiveAccept(from, serverMessageId, timestamp);
531 break;
532 default:
533 break;
534 }
535 }
536
537 void deliverFailedProceed() {
538 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
539 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
540 webRTCWrapper.close();
541 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
542 this.finish();
543 }
544 }
545
546 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
547 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
548 if (originatedFromMyself) {
549 if (transition(State.ACCEPTED)) {
550 if (serverMsgId != null) {
551 this.message.setServerMsgId(serverMsgId);
552 }
553 this.message.setTime(timestamp);
554 this.message.setCarbon(true); //indicate that call was accepted on other device
555 this.writeLogMessageSuccess(0);
556 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
557 this.finish();
558 } else {
559 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
560 }
561 } else {
562 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
563 }
564 }
565
566 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
567 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
568 //reject from another one of my clients
569 if (originatedFromMyself) {
570 receiveRejectFromMyself(serverMsgId, timestamp);
571 } else if (isInitiator()) {
572 if (from.equals(id.with)) {
573 receiveRejectFromResponder();
574 } else {
575 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
576 }
577 } else {
578 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
579 }
580 }
581
582 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
583 if (transition(State.REJECTED)) {
584 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
585 this.finish();
586 if (serverMsgId != null) {
587 this.message.setServerMsgId(serverMsgId);
588 }
589 this.message.setTime(timestamp);
590 this.message.setCarbon(true); //indicate that call was rejected on other device
591 writeLogMessageMissed();
592 } else {
593 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
594 }
595 }
596
597 private void receiveRejectFromResponder() {
598 if (isInState(State.PROCEED)) {
599 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
600 closeTransitionLogFinish(State.REJECTED_RACED);
601 return;
602 }
603 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
604 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
605 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
606 return;
607 }
608 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
609 }
610
611 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
612 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
613 if (originatedFromMyself) {
614 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
615 } else if (transition(State.PROPOSED, () -> {
616 final Collection<RtpDescription> descriptions = Collections2.transform(
617 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
618 input -> (RtpDescription) input
619 );
620 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
621 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
622 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
623 this.proposedMedia = Sets.newHashSet(media);
624 })) {
625 if (serverMsgId != null) {
626 this.message.setServerMsgId(serverMsgId);
627 }
628 this.message.setTime(timestamp);
629 startRinging();
630 } else {
631 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
632 }
633 }
634
635 private void startRinging() {
636 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
637 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
638 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
639 }
640
641 private synchronized void ringingTimeout() {
642 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
643 switch (this.state) {
644 case PROPOSED:
645 message.markUnread();
646 rejectCallFromProposed();
647 break;
648 case SESSION_INITIALIZED:
649 message.markUnread();
650 rejectCallFromSessionInitiate();
651 break;
652 }
653 }
654
655 private void cancelRingingTimeout() {
656 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
657 if (future != null && !future.isCancelled()) {
658 future.cancel(false);
659 }
660 }
661
662 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
663 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
664 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
665 if (from.equals(id.with)) {
666 if (isInitiator()) {
667 if (transition(State.PROCEED)) {
668 if (serverMsgId != null) {
669 this.message.setServerMsgId(serverMsgId);
670 }
671 this.message.setTime(timestamp);
672 final Integer remoteDeviceId = proceed.getDeviceId();
673 if (isOmemoEnabled()) {
674 this.omemoVerification.setDeviceId(remoteDeviceId);
675 } else {
676 if (remoteDeviceId != null) {
677 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": remote party signaled support for OMEMO verification but we have OMEMO disabled");
678 }
679 this.omemoVerification.setDeviceId(null);
680 }
681 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
682 } else {
683 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
684 }
685 } else {
686 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
687 }
688 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
689 if (transition(State.ACCEPTED)) {
690 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
691 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
692 this.finish();
693 }
694 } else {
695 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
696 }
697 }
698
699 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
700 if (from.equals(id.with)) {
701 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
702 if (transition(target)) {
703 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
704 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
705 if (serverMsgId != null) {
706 this.message.setServerMsgId(serverMsgId);
707 }
708 this.message.setTime(timestamp);
709 if (target == State.RETRACTED) {
710 this.message.markUnread();
711 }
712 writeLogMessageMissed();
713 finish();
714 } else {
715 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
716 }
717 } else {
718 //TODO parse retract from self
719 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
720 }
721 }
722
723 public void sendSessionInitiate() {
724 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
725 }
726
727 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
728 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
729 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
730 }
731
732 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
733 if (isTerminated()) {
734 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
735 return;
736 }
737 try {
738 setupWebRTC(media, iceServers);
739 } catch (final WebRTCWrapper.InitializationException e) {
740 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
741 webRTCWrapper.close();
742 sendJingleMessage("retract", id.with.asBareJid());
743 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
744 this.finish();
745 return;
746 }
747 try {
748 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
749 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
750 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
751 sendSessionInitiate(rtpContentMap, targetState);
752 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
753 } catch (final Exception e) {
754 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(e));
755 webRTCWrapper.close();
756 if (isInState(targetState)) {
757 sendSessionTerminate(Reason.FAILED_APPLICATION);
758 } else {
759 sendJingleMessage("retract", id.with.asBareJid());
760 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
761 this.finish();
762 }
763 }
764 }
765
766 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
767 this.initiatorRtpContentMap = rtpContentMap;
768 this.transitionOrThrow(targetState);
769 //TODO do on background thread?
770 final RtpContentMap outgoingContentMap = encryptSessionInitiate(rtpContentMap);
771 final JinglePacket sessionInitiate = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
772 send(sessionInitiate);
773 }
774
775 private RtpContentMap encryptSessionInitiate(final RtpContentMap rtpContentMap) {
776 if (this.omemoVerification.hasDeviceId()) {
777 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
778 try {
779 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
780 } catch (final CryptoFailedException e) {
781 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
782 return rtpContentMap;
783 }
784 this.omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
785 return verifiedPayload.getPayload();
786 } else {
787 return rtpContentMap;
788 }
789 }
790
791 private void sendSessionTerminate(final Reason reason) {
792 sendSessionTerminate(reason, null);
793 }
794
795 private void sendSessionTerminate(final Reason reason, final String text) {
796 final State previous = this.state;
797 final State target = reasonToState(reason);
798 transitionOrThrow(target);
799 if (previous != State.NULL) {
800 writeLogMessage(target);
801 }
802 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
803 jinglePacket.setReason(reason, text);
804 Log.d(Config.LOGTAG, jinglePacket.toString());
805 send(jinglePacket);
806 finish();
807 }
808
809 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
810 final RtpContentMap transportInfo;
811 try {
812 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
813 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
814 } catch (final Exception e) {
815 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
816 return;
817 }
818 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
819 send(jinglePacket);
820 }
821
822 private void send(final JinglePacket jinglePacket) {
823 jinglePacket.setTo(id.with);
824 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
825 }
826
827 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
828 if (response.getType() == IqPacket.TYPE.ERROR) {
829 final String errorCondition = response.getErrorCondition();
830 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
831 if (isTerminated()) {
832 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
833 return;
834 }
835 this.webRTCWrapper.close();
836 final State target;
837 if (Arrays.asList(
838 "service-unavailable",
839 "recipient-unavailable",
840 "remote-server-not-found",
841 "remote-server-timeout"
842 ).contains(errorCondition)) {
843 target = State.TERMINATED_CONNECTIVITY_ERROR;
844 } else {
845 target = State.TERMINATED_APPLICATION_FAILURE;
846 }
847 transitionOrThrow(target);
848 this.finish();
849 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
850 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
851 if (isTerminated()) {
852 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
853 return;
854 }
855 this.webRTCWrapper.close();
856 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
857 this.finish();
858 }
859 }
860
861 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
862 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
863 this.webRTCWrapper.close();
864 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
865 respondWithOutOfOrder(jinglePacket);
866 this.finish();
867 }
868
869 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
870 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
871 }
872
873 private void respondOk(final JinglePacket jinglePacket) {
874 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
875 }
876
877 public void throwStateTransitionException() {
878 final StateTransitionException exception = this.stateTransitionException;
879 if (exception != null) {
880 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
881 }
882 }
883
884 public RtpEndUserState getEndUserState() {
885 switch (this.state) {
886 case NULL:
887 case PROPOSED:
888 case SESSION_INITIALIZED:
889 if (isInitiator()) {
890 return RtpEndUserState.RINGING;
891 } else {
892 return RtpEndUserState.INCOMING_CALL;
893 }
894 case PROCEED:
895 if (isInitiator()) {
896 return RtpEndUserState.RINGING;
897 } else {
898 return RtpEndUserState.ACCEPTING_CALL;
899 }
900 case SESSION_INITIALIZED_PRE_APPROVED:
901 if (isInitiator()) {
902 return RtpEndUserState.RINGING;
903 } else {
904 return RtpEndUserState.CONNECTING;
905 }
906 case SESSION_ACCEPTED:
907 final PeerConnection.PeerConnectionState state;
908 try {
909 state = webRTCWrapper.getState();
910 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
911 //We usually close the WebRTCWrapper *before* transitioning so we might still
912 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
913 return RtpEndUserState.ENDING_CALL;
914 }
915 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
916 return RtpEndUserState.CONNECTED;
917 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
918 return RtpEndUserState.CONNECTING;
919 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
920 return RtpEndUserState.ENDING_CALL;
921 } else {
922 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
923 }
924 case REJECTED:
925 case REJECTED_RACED:
926 case TERMINATED_DECLINED_OR_BUSY:
927 if (isInitiator()) {
928 return RtpEndUserState.DECLINED_OR_BUSY;
929 } else {
930 return RtpEndUserState.ENDED;
931 }
932 case TERMINATED_SUCCESS:
933 case ACCEPTED:
934 case RETRACTED:
935 case TERMINATED_CANCEL_OR_TIMEOUT:
936 return RtpEndUserState.ENDED;
937 case RETRACTED_RACED:
938 if (isInitiator()) {
939 return RtpEndUserState.ENDED;
940 } else {
941 return RtpEndUserState.RETRACTED;
942 }
943 case TERMINATED_CONNECTIVITY_ERROR:
944 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
945 case TERMINATED_APPLICATION_FAILURE:
946 return RtpEndUserState.APPLICATION_ERROR;
947 }
948 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
949 }
950
951 public Set<Media> getMedia() {
952 final State current = getState();
953 if (current == State.NULL) {
954 if (isInitiator()) {
955 return Preconditions.checkNotNull(
956 this.proposedMedia,
957 "RTP connection has not been initialized properly"
958 );
959 }
960 throw new IllegalStateException("RTP connection has not been initialized yet");
961 }
962 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
963 return Preconditions.checkNotNull(
964 this.proposedMedia,
965 "RTP connection has not been initialized properly"
966 );
967 }
968 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
969 if (initiatorContentMap != null) {
970 return initiatorContentMap.getMedia();
971 } else if (isTerminated()) {
972 return Collections.emptySet(); //we might fail before we ever got a chance to set media
973 } else {
974 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
975 }
976 }
977
978
979 public boolean isVerified() {
980 final String fingerprint = this.omemoVerification.getFingerprint();
981 if (fingerprint == null) {
982 return false;
983 }
984 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
985 return status != null && status.getTrust() == FingerprintStatus.Trust.VERIFIED;
986 }
987
988 public synchronized void acceptCall() {
989 switch (this.state) {
990 case PROPOSED:
991 cancelRingingTimeout();
992 acceptCallFromProposed();
993 break;
994 case SESSION_INITIALIZED:
995 cancelRingingTimeout();
996 acceptCallFromSessionInitialized();
997 break;
998 case ACCEPTED:
999 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1000 break;
1001 case PROCEED:
1002 case SESSION_ACCEPTED:
1003 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1004 break;
1005 default:
1006 throw new IllegalStateException("Can not accept call from " + this.state);
1007 }
1008 }
1009
1010
1011 public void notifyPhoneCall() {
1012 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1013 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1014 rejectCall();
1015 } else {
1016 endCall();
1017 }
1018 }
1019
1020 public synchronized void rejectCall() {
1021 if (isTerminated()) {
1022 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1023 return;
1024 }
1025 switch (this.state) {
1026 case PROPOSED:
1027 rejectCallFromProposed();
1028 break;
1029 case SESSION_INITIALIZED:
1030 rejectCallFromSessionInitiate();
1031 break;
1032 default:
1033 throw new IllegalStateException("Can not reject call from " + this.state);
1034 }
1035 }
1036
1037 public synchronized void endCall() {
1038 if (isTerminated()) {
1039 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1040 return;
1041 }
1042 if (isInState(State.PROPOSED) && !isInitiator()) {
1043 rejectCallFromProposed();
1044 return;
1045 }
1046 if (isInState(State.PROCEED)) {
1047 if (isInitiator()) {
1048 retractFromProceed();
1049 } else {
1050 rejectCallFromProceed();
1051 }
1052 return;
1053 }
1054 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1055 this.webRTCWrapper.close();
1056 sendSessionTerminate(Reason.CANCEL);
1057 return;
1058 }
1059 if (isInState(State.SESSION_INITIALIZED)) {
1060 rejectCallFromSessionInitiate();
1061 return;
1062 }
1063 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1064 this.webRTCWrapper.close();
1065 sendSessionTerminate(Reason.SUCCESS);
1066 return;
1067 }
1068 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1069 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1070 return;
1071 }
1072 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1073 }
1074
1075 private void retractFromProceed() {
1076 Log.d(Config.LOGTAG, "retract from proceed");
1077 this.sendJingleMessage("retract");
1078 closeTransitionLogFinish(State.RETRACTED_RACED);
1079 }
1080
1081 private void closeTransitionLogFinish(final State state) {
1082 this.webRTCWrapper.close();
1083 transitionOrThrow(state);
1084 writeLogMessage(state);
1085 finish();
1086 }
1087
1088 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1089 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1090 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1091 if (media.contains(Media.VIDEO)) {
1092 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1093 } else {
1094 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1095 }
1096 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1097 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1098 }
1099
1100 private void acceptCallFromProposed() {
1101 transitionOrThrow(State.PROCEED);
1102 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1103 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1104 this.sendJingleMessage("proceed");
1105 }
1106
1107 private void rejectCallFromProposed() {
1108 transitionOrThrow(State.REJECTED);
1109 writeLogMessageMissed();
1110 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1111 this.sendJingleMessage("reject");
1112 finish();
1113 }
1114
1115 private void rejectCallFromProceed() {
1116 this.sendJingleMessage("reject");
1117 closeTransitionLogFinish(State.REJECTED_RACED);
1118 }
1119
1120 private void rejectCallFromSessionInitiate() {
1121 webRTCWrapper.close();
1122 sendSessionTerminate(Reason.DECLINE);
1123 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1124 }
1125
1126 private void sendJingleMessage(final String action) {
1127 sendJingleMessage(action, id.with);
1128 }
1129
1130 private void sendJingleMessage(final String action, final Jid to) {
1131 final MessagePacket messagePacket = new MessagePacket();
1132 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1133 messagePacket.setTo(to);
1134 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1135 if ("proceed".equals(action)) {
1136 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1137 if (isOmemoEnabled()) {
1138 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1139 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1140 device.setAttribute("id", deviceId);
1141 }
1142 }
1143 messagePacket.addChild("store", "urn:xmpp:hints");
1144 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1145 }
1146
1147 private boolean isOmemoEnabled() {
1148 final Conversational conversational = message.getConversation();
1149 if (conversational instanceof Conversation) {
1150 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1151 }
1152 return false;
1153 }
1154
1155 private void acceptCallFromSessionInitialized() {
1156 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1157 sendSessionAccept();
1158 }
1159
1160 private synchronized boolean isInState(State... state) {
1161 return Arrays.asList(state).contains(this.state);
1162 }
1163
1164 private boolean transition(final State target) {
1165 return transition(target, null);
1166 }
1167
1168 private synchronized boolean transition(final State target, final Runnable runnable) {
1169 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1170 if (validTransitions != null && validTransitions.contains(target)) {
1171 this.state = target;
1172 this.stateTransitionException = new StateTransitionException(target);
1173 if (runnable != null) {
1174 runnable.run();
1175 }
1176 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1177 updateEndUserState();
1178 updateOngoingCallNotification();
1179 return true;
1180 } else {
1181 return false;
1182 }
1183 }
1184
1185 void transitionOrThrow(final State target) {
1186 if (!transition(target)) {
1187 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1188 }
1189 }
1190
1191 @Override
1192 public void onIceCandidate(final IceCandidate iceCandidate) {
1193 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1194 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1195 sendTransportInfo(iceCandidate.sdpMid, candidate);
1196 }
1197
1198 @Override
1199 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1200 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1201 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
1202 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
1203 }
1204 if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) {
1205 this.rtpConnectionEnded = SystemClock.elapsedRealtime();
1206 }
1207 //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace
1208 //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable
1209 //as there is no content-replace
1210 if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) {
1211 if (isTerminated()) {
1212 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1213 return;
1214 }
1215 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1216 } else {
1217 updateEndUserState();
1218 }
1219 }
1220
1221 private void closeWebRTCSessionAfterFailedConnection() {
1222 this.webRTCWrapper.close();
1223 synchronized (this) {
1224 if (isTerminated()) {
1225 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1226 return;
1227 }
1228 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1229 }
1230 }
1231
1232 public long getRtpConnectionStarted() {
1233 return this.rtpConnectionStarted;
1234 }
1235
1236 public long getRtpConnectionEnded() {
1237 return this.rtpConnectionEnded;
1238 }
1239
1240 public AppRTCAudioManager getAudioManager() {
1241 return webRTCWrapper.getAudioManager();
1242 }
1243
1244 public boolean isMicrophoneEnabled() {
1245 return webRTCWrapper.isMicrophoneEnabled();
1246 }
1247
1248 public boolean setMicrophoneEnabled(final boolean enabled) {
1249 return webRTCWrapper.setMicrophoneEnabled(enabled);
1250 }
1251
1252 public boolean isVideoEnabled() {
1253 return webRTCWrapper.isVideoEnabled();
1254 }
1255
1256 public void setVideoEnabled(final boolean enabled) {
1257 webRTCWrapper.setVideoEnabled(enabled);
1258 }
1259
1260 public boolean isCameraSwitchable() {
1261 return webRTCWrapper.isCameraSwitchable();
1262 }
1263
1264 public boolean isFrontCamera() {
1265 return webRTCWrapper.isFrontCamera();
1266 }
1267
1268 public ListenableFuture<Boolean> switchCamera() {
1269 return webRTCWrapper.switchCamera();
1270 }
1271
1272 @Override
1273 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1274 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1275 }
1276
1277 private void updateEndUserState() {
1278 final RtpEndUserState endUserState = getEndUserState();
1279 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1280 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1281 }
1282
1283 private void updateOngoingCallNotification() {
1284 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1285 xmppConnectionService.setOngoingCall(id, getMedia());
1286 } else {
1287 xmppConnectionService.removeOngoingCall();
1288 }
1289 }
1290
1291 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1292 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1293 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1294 request.setTo(id.account.getDomain());
1295 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1296 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1297 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1298 if (response.getType() == IqPacket.TYPE.RESULT) {
1299 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1300 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1301 for (final Element child : children) {
1302 if ("service".equals(child.getName())) {
1303 final String type = child.getAttribute("type");
1304 final String host = child.getAttribute("host");
1305 final String sport = child.getAttribute("port");
1306 final Integer port = sport == null ? null : Ints.tryParse(sport);
1307 final String transport = child.getAttribute("transport");
1308 final String username = child.getAttribute("username");
1309 final String password = child.getAttribute("password");
1310 if (Strings.isNullOrEmpty(host) || port == null) {
1311 continue;
1312 }
1313 if (port < 0 || port > 65535) {
1314 continue;
1315 }
1316 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1317 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1318 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1319 continue;
1320 }
1321 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1322 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1323 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1324 if (username != null && password != null) {
1325 iceServerBuilder.setUsername(username);
1326 iceServerBuilder.setPassword(password);
1327 } else if (Arrays.asList("turn", "turns").contains(type)) {
1328 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1329 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1330 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1331 continue;
1332 }
1333 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1334 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1335 listBuilder.add(iceServer);
1336 }
1337 }
1338 }
1339 }
1340 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1341 if (iceServers.size() == 0) {
1342 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1343 }
1344 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1345 });
1346 } else {
1347 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1348 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1349 }
1350 }
1351
1352 private void finish() {
1353 if (isTerminated()) {
1354 this.cancelRingingTimeout();
1355 this.webRTCWrapper.verifyClosed();
1356 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1357 this.jingleConnectionManager.finishConnectionOrThrow(this);
1358 } else {
1359 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1360 }
1361 }
1362
1363 private void writeLogMessage(final State state) {
1364 final long started = this.rtpConnectionStarted;
1365 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
1366 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1367 writeLogMessageSuccess(duration);
1368 } else {
1369 writeLogMessageMissed();
1370 }
1371 }
1372
1373 private void writeLogMessageSuccess(final long duration) {
1374 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1375 this.writeMessage();
1376 }
1377
1378 private void writeLogMessageMissed() {
1379 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1380 this.writeMessage();
1381 }
1382
1383 private void writeMessage() {
1384 final Conversational conversational = message.getConversation();
1385 if (conversational instanceof Conversation) {
1386 ((Conversation) conversational).add(this.message);
1387 xmppConnectionService.createMessageAsync(message);
1388 xmppConnectionService.updateConversationUi();
1389 } else {
1390 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1391 }
1392 }
1393
1394 public State getState() {
1395 return this.state;
1396 }
1397
1398 boolean isTerminated() {
1399 return TERMINATED.contains(this.state);
1400 }
1401
1402 public Optional<VideoTrack> getLocalVideoTrack() {
1403 return webRTCWrapper.getLocalVideoTrack();
1404 }
1405
1406 public Optional<VideoTrack> getRemoteVideoTrack() {
1407 return webRTCWrapper.getRemoteVideoTrack();
1408 }
1409
1410
1411 public EglBase.Context getEglBaseContext() {
1412 return webRTCWrapper.getEglBaseContext();
1413 }
1414
1415 void setProposedMedia(final Set<Media> media) {
1416 this.proposedMedia = media;
1417 }
1418
1419 public void fireStateUpdate() {
1420 final RtpEndUserState endUserState = getEndUserState();
1421 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1422 }
1423
1424 private interface OnIceServersDiscovered {
1425 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1426 }
1427
1428 private static class StateTransitionException extends Exception {
1429 private final State state;
1430
1431 private StateTransitionException(final State state) {
1432 this.state = state;
1433 }
1434 }
1435}