1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Preconditions;
8import com.google.common.base.Strings;
9import com.google.common.base.Throwables;
10import com.google.common.collect.Collections2;
11import com.google.common.collect.ImmutableList;
12import com.google.common.collect.ImmutableMap;
13import com.google.common.collect.Sets;
14import com.google.common.primitives.Ints;
15import com.google.common.util.concurrent.ListenableFuture;
16
17import org.webrtc.EglBase;
18import org.webrtc.IceCandidate;
19import org.webrtc.PeerConnection;
20import org.webrtc.VideoTrack;
21
22import java.util.ArrayDeque;
23import java.util.Arrays;
24import java.util.Collection;
25import java.util.Collections;
26import java.util.List;
27import java.util.Map;
28import java.util.Set;
29import java.util.concurrent.ScheduledFuture;
30import java.util.concurrent.TimeUnit;
31
32import eu.siacs.conversations.Config;
33import eu.siacs.conversations.crypto.axolotl.AxolotlService;
34import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
35import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
36import eu.siacs.conversations.entities.Account;
37import eu.siacs.conversations.entities.Conversation;
38import eu.siacs.conversations.entities.Conversational;
39import eu.siacs.conversations.entities.Message;
40import eu.siacs.conversations.entities.RtpSessionStatus;
41import eu.siacs.conversations.services.AppRTCAudioManager;
42import eu.siacs.conversations.utils.IP;
43import eu.siacs.conversations.xml.Element;
44import eu.siacs.conversations.xml.Namespace;
45import eu.siacs.conversations.xmpp.Jid;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
57
58 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
59 State.PROCEED,
60 State.SESSION_INITIALIZED_PRE_APPROVED,
61 State.SESSION_ACCEPTED
62 );
63 private static final long BUSY_TIME_OUT = 30;
64 private static final List<State> TERMINATED = Arrays.asList(
65 State.ACCEPTED,
66 State.REJECTED,
67 State.REJECTED_RACED,
68 State.RETRACTED,
69 State.RETRACTED_RACED,
70 State.TERMINATED_SUCCESS,
71 State.TERMINATED_DECLINED_OR_BUSY,
72 State.TERMINATED_CONNECTIVITY_ERROR,
73 State.TERMINATED_CANCEL_OR_TIMEOUT,
74 State.TERMINATED_APPLICATION_FAILURE,
75 State.TERMINATED_SECURITY_ERROR
76 );
77
78 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
79
80 static {
81 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
82 transitionBuilder.put(State.NULL, ImmutableList.of(
83 State.PROPOSED,
84 State.SESSION_INITIALIZED,
85 State.TERMINATED_APPLICATION_FAILURE,
86 State.TERMINATED_SECURITY_ERROR
87 ));
88 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
89 State.ACCEPTED,
90 State.PROCEED,
91 State.REJECTED,
92 State.RETRACTED,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR,
95 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
96 ));
97 transitionBuilder.put(State.PROCEED, ImmutableList.of(
98 State.REJECTED_RACED,
99 State.RETRACTED_RACED,
100 State.SESSION_INITIALIZED_PRE_APPROVED,
101 State.TERMINATED_SUCCESS,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
105 ));
106 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
107 State.SESSION_ACCEPTED,
108 State.TERMINATED_SUCCESS,
109 State.TERMINATED_DECLINED_OR_BUSY,
110 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
111 State.TERMINATED_CANCEL_OR_TIMEOUT,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
125 State.TERMINATED_SUCCESS,
126 State.TERMINATED_DECLINED_OR_BUSY,
127 State.TERMINATED_CONNECTIVITY_ERROR,
128 State.TERMINATED_CANCEL_OR_TIMEOUT,
129 State.TERMINATED_APPLICATION_FAILURE,
130 State.TERMINATED_SECURITY_ERROR
131 ));
132 VALID_TRANSITIONS = transitionBuilder.build();
133 }
134
135 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
136 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
137 private final OmemoVerification omemoVerification = new OmemoVerification();
138 private final Message message;
139 private State state = State.NULL;
140 private StateTransitionException stateTransitionException;
141 private Set<Media> proposedMedia;
142 private RtpContentMap initiatorRtpContentMap;
143 private RtpContentMap responderRtpContentMap;
144 private long rtpConnectionStarted = 0; //time of 'connected'
145 private long rtpConnectionEnded = 0;
146 private ScheduledFuture<?> ringingTimeoutFuture;
147
148 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
149 super(jingleConnectionManager, id, initiator);
150 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
151 id.account,
152 id.with.asBareJid(),
153 false,
154 false
155 );
156 this.message = new Message(
157 conversation,
158 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
159 Message.TYPE_RTP_SESSION,
160 id.sessionId
161 );
162 }
163
164 private static State reasonToState(Reason reason) {
165 switch (reason) {
166 case SUCCESS:
167 return State.TERMINATED_SUCCESS;
168 case DECLINE:
169 case BUSY:
170 return State.TERMINATED_DECLINED_OR_BUSY;
171 case CANCEL:
172 case TIMEOUT:
173 return State.TERMINATED_CANCEL_OR_TIMEOUT;
174 case SECURITY_ERROR:
175 return State.TERMINATED_SECURITY_ERROR;
176 case FAILED_APPLICATION:
177 case UNSUPPORTED_TRANSPORTS:
178 case UNSUPPORTED_APPLICATIONS:
179 return State.TERMINATED_APPLICATION_FAILURE;
180 default:
181 return State.TERMINATED_CONNECTIVITY_ERROR;
182 }
183 }
184
185 @Override
186 synchronized void deliverPacket(final JinglePacket jinglePacket) {
187 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
188 switch (jinglePacket.getAction()) {
189 case SESSION_INITIATE:
190 receiveSessionInitiate(jinglePacket);
191 break;
192 case TRANSPORT_INFO:
193 receiveTransportInfo(jinglePacket);
194 break;
195 case SESSION_ACCEPT:
196 receiveSessionAccept(jinglePacket);
197 break;
198 case SESSION_TERMINATE:
199 receiveSessionTerminate(jinglePacket);
200 break;
201 default:
202 respondOk(jinglePacket);
203 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
204 break;
205 }
206 }
207
208 @Override
209 synchronized void notifyRebound() {
210 if (isTerminated()) {
211 return;
212 }
213 webRTCWrapper.close();
214 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
215 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
216 }
217 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
218 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
219 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
220 } else {
221 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
222 finish();
223 }
224 }
225
226 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
227 respondOk(jinglePacket);
228 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
229 final State previous = this.state;
230 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
231 if (TERMINATED.contains(previous)) {
232 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
233 return;
234 }
235 webRTCWrapper.close();
236 final State target = reasonToState(wrapper.reason);
237 transitionOrThrow(target);
238 writeLogMessage(target);
239 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
240 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
241 }
242 finish();
243 }
244
245 private void receiveTransportInfo(final JinglePacket jinglePacket) {
246 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
247 respondOk(jinglePacket);
248 final RtpContentMap contentMap;
249 try {
250 contentMap = RtpContentMap.of(jinglePacket);
251 } catch (IllegalArgumentException | NullPointerException e) {
252 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
253 return;
254 }
255 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
256 if (this.state == State.SESSION_ACCEPTED) {
257 try {
258 processCandidates(candidates);
259 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
260 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
261 }
262 } else {
263 pendingIceCandidates.push(candidates);
264 }
265 } else {
266 if (isTerminated()) {
267 respondOk(jinglePacket);
268 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
269 } else {
270 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
271 terminateWithOutOfOrder(jinglePacket);
272 }
273 }
274 }
275
276 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
277 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
278 final Group originalGroup = rtpContentMap.group;
279 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
280 if (identificationTags.size() == 0) {
281 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
282 }
283 processCandidates(identificationTags, contents);
284 }
285
286 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
287 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
288 final String ufrag = content.getValue().transport.getAttribute("ufrag");
289 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
290 final String sdp;
291 try {
292 sdp = candidate.toSdpAttribute(ufrag);
293 } catch (IllegalArgumentException e) {
294 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
295 continue;
296 }
297 final String sdpMid = content.getKey();
298 final int mLineIndex = indices.indexOf(sdpMid);
299 if (mLineIndex < 0) {
300 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
301 }
302 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
303 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
304 this.webRTCWrapper.addIceCandidate(iceCandidate);
305 }
306 }
307 }
308
309 private RtpContentMap receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
310 final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket);
311 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
312 final AxolotlService.OmemoVerifiedPayload<RtpContentMap> omemoVerifiedPayload;
313 try {
314 omemoVerifiedPayload = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
315 } catch (final CryptoFailedException e) {
316 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
317 }
318 this.omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
319 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + this.omemoVerification);
320 return omemoVerifiedPayload.getPayload();
321 } else if (expectVerification) {
322 throw new SecurityException("DTLS fingerprint was unexpectedly not verifiable");
323 } else {
324 return receivedContentMap;
325 }
326 }
327
328 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
329 if (isInitiator()) {
330 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
331 if (isTerminated()) {
332 Log.d(Config.LOGTAG, String.format(
333 "%s: got a reason to terminate with out-of-order. but already in state %s",
334 id.account.getJid().asBareJid(),
335 getState()
336 ));
337 respondWithOutOfOrder(jinglePacket);
338 } else {
339 terminateWithOutOfOrder(jinglePacket);
340 }
341 return;
342 }
343 final RtpContentMap contentMap;
344 try {
345 contentMap = receiveRtpContentMap(jinglePacket, false);
346 contentMap.requireContentDescriptions();
347 contentMap.requireDTLSFingerprint();
348 } catch (final RuntimeException e) {
349 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
350 respondOk(jinglePacket);
351 sendSessionTerminate(Reason.of(e), e.getMessage());
352 return;
353 }
354 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
355 final State target;
356 if (this.state == State.PROCEED) {
357 Preconditions.checkState(
358 proposedMedia != null && proposedMedia.size() > 0,
359 "proposed media must be set when processing pre-approved session-initiate"
360 );
361 if (!this.proposedMedia.equals(contentMap.getMedia())) {
362 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
363 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
364 this.proposedMedia,
365 contentMap.getMedia()
366 ));
367 return;
368 }
369 target = State.SESSION_INITIALIZED_PRE_APPROVED;
370 } else {
371 target = State.SESSION_INITIALIZED;
372 }
373 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
374 respondOk(jinglePacket);
375
376 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
377 if (candidates.size() > 0) {
378 pendingIceCandidates.push(candidates);
379 }
380 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
381 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
382 sendSessionAccept();
383 } else {
384 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
385 startRinging();
386 }
387 } else {
388 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
389 terminateWithOutOfOrder(jinglePacket);
390 }
391 }
392
393 private void receiveSessionAccept(final JinglePacket jinglePacket) {
394 if (!isInitiator()) {
395 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
396 terminateWithOutOfOrder(jinglePacket);
397 return;
398 }
399 final RtpContentMap contentMap;
400 try {
401 contentMap = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
402 contentMap.requireContentDescriptions();
403 contentMap.requireDTLSFingerprint();
404 } catch (final RuntimeException e) {
405 respondOk(jinglePacket);
406 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
407 webRTCWrapper.close();
408 sendSessionTerminate(Reason.of(e), e.getMessage());
409 return;
410 }
411 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
412 if (!initiatorMedia.equals(contentMap.getMedia())) {
413 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
414 "Your session-included included media %s but our session-initiate was %s",
415 this.proposedMedia,
416 contentMap.getMedia()
417 ));
418 return;
419 }
420 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
421 if (transition(State.SESSION_ACCEPTED)) {
422 respondOk(jinglePacket);
423 receiveSessionAccept(contentMap);
424 } else {
425 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
426 respondOk(jinglePacket);
427 }
428 }
429
430 private void receiveSessionAccept(final RtpContentMap contentMap) {
431 this.responderRtpContentMap = contentMap;
432 final SessionDescription sessionDescription;
433 try {
434 sessionDescription = SessionDescription.of(contentMap);
435 } catch (final IllegalArgumentException | NullPointerException e) {
436 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
437 webRTCWrapper.close();
438 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
439 return;
440 }
441 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
442 org.webrtc.SessionDescription.Type.ANSWER,
443 sessionDescription.toString()
444 );
445 try {
446 this.webRTCWrapper.setRemoteDescription(answer).get();
447 } catch (final Exception e) {
448 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
449 webRTCWrapper.close();
450 sendSessionTerminate(Reason.FAILED_APPLICATION);
451 return;
452 }
453 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
454 processCandidates(identificationTags, contentMap.contents.entrySet());
455 }
456
457 private void sendSessionAccept() {
458 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
459 if (rtpContentMap == null) {
460 throw new IllegalStateException("initiator RTP Content Map has not been set");
461 }
462 final SessionDescription offer;
463 try {
464 offer = SessionDescription.of(rtpContentMap);
465 } catch (final IllegalArgumentException | NullPointerException e) {
466 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
467 webRTCWrapper.close();
468 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
469 return;
470 }
471 sendSessionAccept(rtpContentMap.getMedia(), offer);
472 }
473
474 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
475 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
476 }
477
478 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
479 if (isTerminated()) {
480 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
481 return;
482 }
483 try {
484 setupWebRTC(media, iceServers);
485 } catch (final WebRTCWrapper.InitializationException e) {
486 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
487 webRTCWrapper.close();
488 sendSessionTerminate(Reason.FAILED_APPLICATION);
489 return;
490 }
491 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
492 org.webrtc.SessionDescription.Type.OFFER,
493 offer.toString()
494 );
495 try {
496 this.webRTCWrapper.setRemoteDescription(sdp).get();
497 addIceCandidatesFromBlackLog();
498 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
499 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
500 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
501 sendSessionAccept(respondingRtpContentMap);
502 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
503 } catch (final Exception e) {
504 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(e));
505 webRTCWrapper.close();
506 sendSessionTerminate(Reason.FAILED_APPLICATION);
507 }
508 }
509
510 private void addIceCandidatesFromBlackLog() {
511 while (!this.pendingIceCandidates.isEmpty()) {
512 processCandidates(this.pendingIceCandidates.poll());
513 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
514 }
515 }
516
517 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
518 this.responderRtpContentMap = rtpContentMap;
519 this.transitionOrThrow(State.SESSION_ACCEPTED);
520 final RtpContentMap outgoingContentMap;
521 if (this.omemoVerification.hasDeviceId()) {
522 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
523 try {
524 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
525 outgoingContentMap = verifiedPayload.getPayload();
526 this.omemoVerification.setOrEnsureEqual(verifiedPayload);
527 } catch (final Exception e) {
528 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
529 }
530 } else {
531 outgoingContentMap = rtpContentMap;
532 }
533 final JinglePacket sessionAccept = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
534 send(sessionAccept);
535 }
536
537 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
538 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
539 switch (message.getName()) {
540 case "propose":
541 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
542 break;
543 case "proceed":
544 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
545 break;
546 case "retract":
547 receiveRetract(from, serverMessageId, timestamp);
548 break;
549 case "reject":
550 receiveReject(from, serverMessageId, timestamp);
551 break;
552 case "accept":
553 receiveAccept(from, serverMessageId, timestamp);
554 break;
555 default:
556 break;
557 }
558 }
559
560 void deliverFailedProceed() {
561 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
562 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
563 webRTCWrapper.close();
564 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
565 this.finish();
566 }
567 }
568
569 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
570 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
571 if (originatedFromMyself) {
572 if (transition(State.ACCEPTED)) {
573 if (serverMsgId != null) {
574 this.message.setServerMsgId(serverMsgId);
575 }
576 this.message.setTime(timestamp);
577 this.message.setCarbon(true); //indicate that call was accepted on other device
578 this.writeLogMessageSuccess(0);
579 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
580 this.finish();
581 } else {
582 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
583 }
584 } else {
585 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
586 }
587 }
588
589 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
590 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
591 //reject from another one of my clients
592 if (originatedFromMyself) {
593 receiveRejectFromMyself(serverMsgId, timestamp);
594 } else if (isInitiator()) {
595 if (from.equals(id.with)) {
596 receiveRejectFromResponder();
597 } else {
598 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
599 }
600 } else {
601 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
602 }
603 }
604
605 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
606 if (transition(State.REJECTED)) {
607 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
608 this.finish();
609 if (serverMsgId != null) {
610 this.message.setServerMsgId(serverMsgId);
611 }
612 this.message.setTime(timestamp);
613 this.message.setCarbon(true); //indicate that call was rejected on other device
614 writeLogMessageMissed();
615 } else {
616 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
617 }
618 }
619
620 private void receiveRejectFromResponder() {
621 if (isInState(State.PROCEED)) {
622 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
623 closeTransitionLogFinish(State.REJECTED_RACED);
624 return;
625 }
626 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
627 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
628 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
629 return;
630 }
631 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
632 }
633
634 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
635 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
636 if (originatedFromMyself) {
637 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
638 } else if (transition(State.PROPOSED, () -> {
639 final Collection<RtpDescription> descriptions = Collections2.transform(
640 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
641 input -> (RtpDescription) input
642 );
643 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
644 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
645 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
646 this.proposedMedia = Sets.newHashSet(media);
647 })) {
648 if (serverMsgId != null) {
649 this.message.setServerMsgId(serverMsgId);
650 }
651 this.message.setTime(timestamp);
652 startRinging();
653 } else {
654 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
655 }
656 }
657
658 private void startRinging() {
659 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
660 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
661 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
662 }
663
664 private synchronized void ringingTimeout() {
665 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
666 switch (this.state) {
667 case PROPOSED:
668 message.markUnread();
669 rejectCallFromProposed();
670 break;
671 case SESSION_INITIALIZED:
672 message.markUnread();
673 rejectCallFromSessionInitiate();
674 break;
675 }
676 }
677
678 private void cancelRingingTimeout() {
679 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
680 if (future != null && !future.isCancelled()) {
681 future.cancel(false);
682 }
683 }
684
685 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
686 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
687 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
688 if (from.equals(id.with)) {
689 if (isInitiator()) {
690 if (transition(State.PROCEED)) {
691 if (serverMsgId != null) {
692 this.message.setServerMsgId(serverMsgId);
693 }
694 this.message.setTime(timestamp);
695 final Integer remoteDeviceId = proceed.getDeviceId();
696 if (isOmemoEnabled()) {
697 this.omemoVerification.setDeviceId(remoteDeviceId);
698 } else {
699 if (remoteDeviceId != null) {
700 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
701 }
702 this.omemoVerification.setDeviceId(null);
703 }
704 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
705 } else {
706 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
707 }
708 } else {
709 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
710 }
711 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
712 if (transition(State.ACCEPTED)) {
713 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
714 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
715 this.finish();
716 }
717 } else {
718 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
719 }
720 }
721
722 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
723 if (from.equals(id.with)) {
724 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
725 if (transition(target)) {
726 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
727 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
728 if (serverMsgId != null) {
729 this.message.setServerMsgId(serverMsgId);
730 }
731 this.message.setTime(timestamp);
732 if (target == State.RETRACTED) {
733 this.message.markUnread();
734 }
735 writeLogMessageMissed();
736 finish();
737 } else {
738 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
739 }
740 } else {
741 //TODO parse retract from self
742 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
743 }
744 }
745
746 public void sendSessionInitiate() {
747 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
748 }
749
750 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
751 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
752 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
753 }
754
755 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
756 if (isTerminated()) {
757 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
758 return;
759 }
760 try {
761 setupWebRTC(media, iceServers);
762 } catch (final WebRTCWrapper.InitializationException e) {
763 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
764 webRTCWrapper.close();
765 sendJingleMessage("retract", id.with.asBareJid());
766 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
767 this.finish();
768 return;
769 }
770 try {
771 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
772 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
773 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
774 sendSessionInitiate(rtpContentMap, targetState);
775 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
776 } catch (final Exception e) {
777 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(e));
778 webRTCWrapper.close();
779 if (isInState(targetState)) {
780 sendSessionTerminate(Reason.FAILED_APPLICATION);
781 } else {
782 sendJingleMessage("retract", id.with.asBareJid());
783 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
784 this.finish();
785 }
786 }
787 }
788
789 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
790 this.initiatorRtpContentMap = rtpContentMap;
791 this.transitionOrThrow(targetState);
792 //TODO do on background thread?
793 final RtpContentMap outgoingContentMap = encryptSessionInitiate(rtpContentMap);
794 final JinglePacket sessionInitiate = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
795 send(sessionInitiate);
796 }
797
798 private RtpContentMap encryptSessionInitiate(final RtpContentMap rtpContentMap) {
799 if (this.omemoVerification.hasDeviceId()) {
800 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
801 try {
802 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
803 } catch (final CryptoFailedException e) {
804 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
805 return rtpContentMap;
806 }
807 this.omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
808 return verifiedPayload.getPayload();
809 } else {
810 return rtpContentMap;
811 }
812 }
813
814 private void sendSessionTerminate(final Reason reason) {
815 sendSessionTerminate(reason, null);
816 }
817
818 private void sendSessionTerminate(final Reason reason, final String text) {
819 final State previous = this.state;
820 final State target = reasonToState(reason);
821 transitionOrThrow(target);
822 if (previous != State.NULL) {
823 writeLogMessage(target);
824 }
825 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
826 jinglePacket.setReason(reason, text);
827 Log.d(Config.LOGTAG, jinglePacket.toString());
828 send(jinglePacket);
829 finish();
830 }
831
832 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
833 final RtpContentMap transportInfo;
834 try {
835 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
836 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
837 } catch (final Exception e) {
838 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
839 return;
840 }
841 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
842 send(jinglePacket);
843 }
844
845 private void send(final JinglePacket jinglePacket) {
846 jinglePacket.setTo(id.with);
847 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
848 }
849
850 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
851 if (response.getType() == IqPacket.TYPE.ERROR) {
852 final String errorCondition = response.getErrorCondition();
853 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
854 if (isTerminated()) {
855 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
856 return;
857 }
858 this.webRTCWrapper.close();
859 final State target;
860 if (Arrays.asList(
861 "service-unavailable",
862 "recipient-unavailable",
863 "remote-server-not-found",
864 "remote-server-timeout"
865 ).contains(errorCondition)) {
866 target = State.TERMINATED_CONNECTIVITY_ERROR;
867 } else {
868 target = State.TERMINATED_APPLICATION_FAILURE;
869 }
870 transitionOrThrow(target);
871 this.finish();
872 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
873 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
874 if (isTerminated()) {
875 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
876 return;
877 }
878 this.webRTCWrapper.close();
879 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
880 this.finish();
881 }
882 }
883
884 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
885 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
886 this.webRTCWrapper.close();
887 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
888 respondWithOutOfOrder(jinglePacket);
889 this.finish();
890 }
891
892 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
893 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
894 }
895
896 private void respondOk(final JinglePacket jinglePacket) {
897 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
898 }
899
900 public void throwStateTransitionException() {
901 final StateTransitionException exception = this.stateTransitionException;
902 if (exception != null) {
903 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
904 }
905 }
906
907 public RtpEndUserState getEndUserState() {
908 switch (this.state) {
909 case NULL:
910 case PROPOSED:
911 case SESSION_INITIALIZED:
912 if (isInitiator()) {
913 return RtpEndUserState.RINGING;
914 } else {
915 return RtpEndUserState.INCOMING_CALL;
916 }
917 case PROCEED:
918 if (isInitiator()) {
919 return RtpEndUserState.RINGING;
920 } else {
921 return RtpEndUserState.ACCEPTING_CALL;
922 }
923 case SESSION_INITIALIZED_PRE_APPROVED:
924 if (isInitiator()) {
925 return RtpEndUserState.RINGING;
926 } else {
927 return RtpEndUserState.CONNECTING;
928 }
929 case SESSION_ACCEPTED:
930 final PeerConnection.PeerConnectionState state;
931 try {
932 state = webRTCWrapper.getState();
933 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
934 //We usually close the WebRTCWrapper *before* transitioning so we might still
935 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
936 return RtpEndUserState.ENDING_CALL;
937 }
938 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
939 return RtpEndUserState.CONNECTED;
940 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
941 return RtpEndUserState.CONNECTING;
942 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
943 return RtpEndUserState.ENDING_CALL;
944 } else {
945 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
946 }
947 case REJECTED:
948 case REJECTED_RACED:
949 case TERMINATED_DECLINED_OR_BUSY:
950 if (isInitiator()) {
951 return RtpEndUserState.DECLINED_OR_BUSY;
952 } else {
953 return RtpEndUserState.ENDED;
954 }
955 case TERMINATED_SUCCESS:
956 case ACCEPTED:
957 case RETRACTED:
958 case TERMINATED_CANCEL_OR_TIMEOUT:
959 return RtpEndUserState.ENDED;
960 case RETRACTED_RACED:
961 if (isInitiator()) {
962 return RtpEndUserState.ENDED;
963 } else {
964 return RtpEndUserState.RETRACTED;
965 }
966 case TERMINATED_CONNECTIVITY_ERROR:
967 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
968 case TERMINATED_APPLICATION_FAILURE:
969 return RtpEndUserState.APPLICATION_ERROR;
970 case TERMINATED_SECURITY_ERROR:
971 return RtpEndUserState.SECURITY_ERROR;
972 }
973 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
974 }
975
976 public Set<Media> getMedia() {
977 final State current = getState();
978 if (current == State.NULL) {
979 if (isInitiator()) {
980 return Preconditions.checkNotNull(
981 this.proposedMedia,
982 "RTP connection has not been initialized properly"
983 );
984 }
985 throw new IllegalStateException("RTP connection has not been initialized yet");
986 }
987 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
988 return Preconditions.checkNotNull(
989 this.proposedMedia,
990 "RTP connection has not been initialized properly"
991 );
992 }
993 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
994 if (initiatorContentMap != null) {
995 return initiatorContentMap.getMedia();
996 } else if (isTerminated()) {
997 return Collections.emptySet(); //we might fail before we ever got a chance to set media
998 } else {
999 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1000 }
1001 }
1002
1003
1004 public boolean isVerified() {
1005 final String fingerprint = this.omemoVerification.getFingerprint();
1006 if (fingerprint == null) {
1007 return false;
1008 }
1009 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1010 return status != null && status.getTrust() == FingerprintStatus.Trust.VERIFIED;
1011 }
1012
1013 public synchronized void acceptCall() {
1014 switch (this.state) {
1015 case PROPOSED:
1016 cancelRingingTimeout();
1017 acceptCallFromProposed();
1018 break;
1019 case SESSION_INITIALIZED:
1020 cancelRingingTimeout();
1021 acceptCallFromSessionInitialized();
1022 break;
1023 case ACCEPTED:
1024 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1025 break;
1026 case PROCEED:
1027 case SESSION_ACCEPTED:
1028 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1029 break;
1030 default:
1031 throw new IllegalStateException("Can not accept call from " + this.state);
1032 }
1033 }
1034
1035
1036 public void notifyPhoneCall() {
1037 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1038 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1039 rejectCall();
1040 } else {
1041 endCall();
1042 }
1043 }
1044
1045 public synchronized void rejectCall() {
1046 if (isTerminated()) {
1047 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1048 return;
1049 }
1050 switch (this.state) {
1051 case PROPOSED:
1052 rejectCallFromProposed();
1053 break;
1054 case SESSION_INITIALIZED:
1055 rejectCallFromSessionInitiate();
1056 break;
1057 default:
1058 throw new IllegalStateException("Can not reject call from " + this.state);
1059 }
1060 }
1061
1062 public synchronized void endCall() {
1063 if (isTerminated()) {
1064 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1065 return;
1066 }
1067 if (isInState(State.PROPOSED) && !isInitiator()) {
1068 rejectCallFromProposed();
1069 return;
1070 }
1071 if (isInState(State.PROCEED)) {
1072 if (isInitiator()) {
1073 retractFromProceed();
1074 } else {
1075 rejectCallFromProceed();
1076 }
1077 return;
1078 }
1079 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1080 this.webRTCWrapper.close();
1081 sendSessionTerminate(Reason.CANCEL);
1082 return;
1083 }
1084 if (isInState(State.SESSION_INITIALIZED)) {
1085 rejectCallFromSessionInitiate();
1086 return;
1087 }
1088 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1089 this.webRTCWrapper.close();
1090 sendSessionTerminate(Reason.SUCCESS);
1091 return;
1092 }
1093 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1094 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1095 return;
1096 }
1097 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1098 }
1099
1100 private void retractFromProceed() {
1101 Log.d(Config.LOGTAG, "retract from proceed");
1102 this.sendJingleMessage("retract");
1103 closeTransitionLogFinish(State.RETRACTED_RACED);
1104 }
1105
1106 private void closeTransitionLogFinish(final State state) {
1107 this.webRTCWrapper.close();
1108 transitionOrThrow(state);
1109 writeLogMessage(state);
1110 finish();
1111 }
1112
1113 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1114 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1115 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1116 if (media.contains(Media.VIDEO)) {
1117 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1118 } else {
1119 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1120 }
1121 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1122 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1123 }
1124
1125 private void acceptCallFromProposed() {
1126 transitionOrThrow(State.PROCEED);
1127 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1128 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1129 this.sendJingleMessage("proceed");
1130 }
1131
1132 private void rejectCallFromProposed() {
1133 transitionOrThrow(State.REJECTED);
1134 writeLogMessageMissed();
1135 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1136 this.sendJingleMessage("reject");
1137 finish();
1138 }
1139
1140 private void rejectCallFromProceed() {
1141 this.sendJingleMessage("reject");
1142 closeTransitionLogFinish(State.REJECTED_RACED);
1143 }
1144
1145 private void rejectCallFromSessionInitiate() {
1146 webRTCWrapper.close();
1147 sendSessionTerminate(Reason.DECLINE);
1148 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1149 }
1150
1151 private void sendJingleMessage(final String action) {
1152 sendJingleMessage(action, id.with);
1153 }
1154
1155 private void sendJingleMessage(final String action, final Jid to) {
1156 final MessagePacket messagePacket = new MessagePacket();
1157 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1158 messagePacket.setTo(to);
1159 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1160 if ("proceed".equals(action)) {
1161 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1162 if (isOmemoEnabled()) {
1163 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1164 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1165 device.setAttribute("id", deviceId);
1166 }
1167 }
1168 messagePacket.addChild("store", "urn:xmpp:hints");
1169 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1170 }
1171
1172 private boolean isOmemoEnabled() {
1173 final Conversational conversational = message.getConversation();
1174 if (conversational instanceof Conversation) {
1175 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1176 }
1177 return false;
1178 }
1179
1180 private void acceptCallFromSessionInitialized() {
1181 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1182 sendSessionAccept();
1183 }
1184
1185 private synchronized boolean isInState(State... state) {
1186 return Arrays.asList(state).contains(this.state);
1187 }
1188
1189 private boolean transition(final State target) {
1190 return transition(target, null);
1191 }
1192
1193 private synchronized boolean transition(final State target, final Runnable runnable) {
1194 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1195 if (validTransitions != null && validTransitions.contains(target)) {
1196 this.state = target;
1197 this.stateTransitionException = new StateTransitionException(target);
1198 if (runnable != null) {
1199 runnable.run();
1200 }
1201 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1202 updateEndUserState();
1203 updateOngoingCallNotification();
1204 return true;
1205 } else {
1206 return false;
1207 }
1208 }
1209
1210 void transitionOrThrow(final State target) {
1211 if (!transition(target)) {
1212 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1213 }
1214 }
1215
1216 @Override
1217 public void onIceCandidate(final IceCandidate iceCandidate) {
1218 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1219 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1220 sendTransportInfo(iceCandidate.sdpMid, candidate);
1221 }
1222
1223 @Override
1224 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1225 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1226 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
1227 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
1228 }
1229 if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) {
1230 this.rtpConnectionEnded = SystemClock.elapsedRealtime();
1231 }
1232 //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace
1233 //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable
1234 //as there is no content-replace
1235 if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) {
1236 if (isTerminated()) {
1237 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1238 return;
1239 }
1240 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1241 } else {
1242 updateEndUserState();
1243 }
1244 }
1245
1246 private void closeWebRTCSessionAfterFailedConnection() {
1247 this.webRTCWrapper.close();
1248 synchronized (this) {
1249 if (isTerminated()) {
1250 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1251 return;
1252 }
1253 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1254 }
1255 }
1256
1257 public long getRtpConnectionStarted() {
1258 return this.rtpConnectionStarted;
1259 }
1260
1261 public long getRtpConnectionEnded() {
1262 return this.rtpConnectionEnded;
1263 }
1264
1265 public AppRTCAudioManager getAudioManager() {
1266 return webRTCWrapper.getAudioManager();
1267 }
1268
1269 public boolean isMicrophoneEnabled() {
1270 return webRTCWrapper.isMicrophoneEnabled();
1271 }
1272
1273 public boolean setMicrophoneEnabled(final boolean enabled) {
1274 return webRTCWrapper.setMicrophoneEnabled(enabled);
1275 }
1276
1277 public boolean isVideoEnabled() {
1278 return webRTCWrapper.isVideoEnabled();
1279 }
1280
1281 public void setVideoEnabled(final boolean enabled) {
1282 webRTCWrapper.setVideoEnabled(enabled);
1283 }
1284
1285 public boolean isCameraSwitchable() {
1286 return webRTCWrapper.isCameraSwitchable();
1287 }
1288
1289 public boolean isFrontCamera() {
1290 return webRTCWrapper.isFrontCamera();
1291 }
1292
1293 public ListenableFuture<Boolean> switchCamera() {
1294 return webRTCWrapper.switchCamera();
1295 }
1296
1297 @Override
1298 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1299 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1300 }
1301
1302 private void updateEndUserState() {
1303 final RtpEndUserState endUserState = getEndUserState();
1304 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1305 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1306 }
1307
1308 private void updateOngoingCallNotification() {
1309 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1310 xmppConnectionService.setOngoingCall(id, getMedia());
1311 } else {
1312 xmppConnectionService.removeOngoingCall();
1313 }
1314 }
1315
1316 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1317 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1318 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1319 request.setTo(id.account.getDomain());
1320 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1321 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1322 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1323 if (response.getType() == IqPacket.TYPE.RESULT) {
1324 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1325 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1326 for (final Element child : children) {
1327 if ("service".equals(child.getName())) {
1328 final String type = child.getAttribute("type");
1329 final String host = child.getAttribute("host");
1330 final String sport = child.getAttribute("port");
1331 final Integer port = sport == null ? null : Ints.tryParse(sport);
1332 final String transport = child.getAttribute("transport");
1333 final String username = child.getAttribute("username");
1334 final String password = child.getAttribute("password");
1335 if (Strings.isNullOrEmpty(host) || port == null) {
1336 continue;
1337 }
1338 if (port < 0 || port > 65535) {
1339 continue;
1340 }
1341 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1342 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1343 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1344 continue;
1345 }
1346 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1347 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1348 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1349 if (username != null && password != null) {
1350 iceServerBuilder.setUsername(username);
1351 iceServerBuilder.setPassword(password);
1352 } else if (Arrays.asList("turn", "turns").contains(type)) {
1353 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1354 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1355 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1356 continue;
1357 }
1358 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1359 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1360 listBuilder.add(iceServer);
1361 }
1362 }
1363 }
1364 }
1365 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1366 if (iceServers.size() == 0) {
1367 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1368 }
1369 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1370 });
1371 } else {
1372 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1373 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1374 }
1375 }
1376
1377 private void finish() {
1378 if (isTerminated()) {
1379 this.cancelRingingTimeout();
1380 this.webRTCWrapper.verifyClosed();
1381 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1382 this.jingleConnectionManager.finishConnectionOrThrow(this);
1383 } else {
1384 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1385 }
1386 }
1387
1388 private void writeLogMessage(final State state) {
1389 final long started = this.rtpConnectionStarted;
1390 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
1391 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1392 writeLogMessageSuccess(duration);
1393 } else {
1394 writeLogMessageMissed();
1395 }
1396 }
1397
1398 private void writeLogMessageSuccess(final long duration) {
1399 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1400 this.writeMessage();
1401 }
1402
1403 private void writeLogMessageMissed() {
1404 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1405 this.writeMessage();
1406 }
1407
1408 private void writeMessage() {
1409 final Conversational conversational = message.getConversation();
1410 if (conversational instanceof Conversation) {
1411 ((Conversation) conversational).add(this.message);
1412 xmppConnectionService.createMessageAsync(message);
1413 xmppConnectionService.updateConversationUi();
1414 } else {
1415 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1416 }
1417 }
1418
1419 public State getState() {
1420 return this.state;
1421 }
1422
1423 boolean isTerminated() {
1424 return TERMINATED.contains(this.state);
1425 }
1426
1427 public Optional<VideoTrack> getLocalVideoTrack() {
1428 return webRTCWrapper.getLocalVideoTrack();
1429 }
1430
1431 public Optional<VideoTrack> getRemoteVideoTrack() {
1432 return webRTCWrapper.getRemoteVideoTrack();
1433 }
1434
1435
1436 public EglBase.Context getEglBaseContext() {
1437 return webRTCWrapper.getEglBaseContext();
1438 }
1439
1440 void setProposedMedia(final Set<Media> media) {
1441 this.proposedMedia = media;
1442 }
1443
1444 public void fireStateUpdate() {
1445 final RtpEndUserState endUserState = getEndUserState();
1446 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1447 }
1448
1449 private interface OnIceServersDiscovered {
1450 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1451 }
1452
1453 private static class StateTransitionException extends Exception {
1454 private final State state;
1455
1456 private StateTransitionException(final State state) {
1457 this.state = state;
1458 }
1459 }
1460}