1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Preconditions;
8import com.google.common.base.Strings;
9import com.google.common.base.Throwables;
10import com.google.common.collect.Collections2;
11import com.google.common.collect.ImmutableList;
12import com.google.common.collect.ImmutableMap;
13import com.google.common.collect.Sets;
14import com.google.common.primitives.Ints;
15import com.google.common.util.concurrent.ListenableFuture;
16
17import org.webrtc.EglBase;
18import org.webrtc.IceCandidate;
19import org.webrtc.PeerConnection;
20import org.webrtc.VideoTrack;
21
22import java.util.ArrayDeque;
23import java.util.Arrays;
24import java.util.Collection;
25import java.util.Collections;
26import java.util.List;
27import java.util.Map;
28import java.util.Set;
29import java.util.concurrent.ScheduledFuture;
30import java.util.concurrent.TimeUnit;
31
32import eu.siacs.conversations.Config;
33import eu.siacs.conversations.crypto.axolotl.AxolotlService;
34import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
35import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
36import eu.siacs.conversations.entities.Account;
37import eu.siacs.conversations.entities.Conversation;
38import eu.siacs.conversations.entities.Conversational;
39import eu.siacs.conversations.entities.Message;
40import eu.siacs.conversations.entities.RtpSessionStatus;
41import eu.siacs.conversations.services.AppRTCAudioManager;
42import eu.siacs.conversations.utils.IP;
43import eu.siacs.conversations.xml.Element;
44import eu.siacs.conversations.xml.Namespace;
45import eu.siacs.conversations.xmpp.Jid;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
57
58 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
59 State.PROCEED,
60 State.SESSION_INITIALIZED_PRE_APPROVED,
61 State.SESSION_ACCEPTED
62 );
63 private static final long BUSY_TIME_OUT = 30;
64 private static final List<State> TERMINATED = Arrays.asList(
65 State.ACCEPTED,
66 State.REJECTED,
67 State.REJECTED_RACED,
68 State.RETRACTED,
69 State.RETRACTED_RACED,
70 State.TERMINATED_SUCCESS,
71 State.TERMINATED_DECLINED_OR_BUSY,
72 State.TERMINATED_CONNECTIVITY_ERROR,
73 State.TERMINATED_CANCEL_OR_TIMEOUT,
74 State.TERMINATED_APPLICATION_FAILURE,
75 State.TERMINATED_SECURITY_ERROR
76 );
77
78 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
79
80 static {
81 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
82 transitionBuilder.put(State.NULL, ImmutableList.of(
83 State.PROPOSED,
84 State.SESSION_INITIALIZED,
85 State.TERMINATED_APPLICATION_FAILURE,
86 State.TERMINATED_SECURITY_ERROR
87 ));
88 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
89 State.ACCEPTED,
90 State.PROCEED,
91 State.REJECTED,
92 State.RETRACTED,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR,
95 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
96 ));
97 transitionBuilder.put(State.PROCEED, ImmutableList.of(
98 State.REJECTED_RACED,
99 State.RETRACTED_RACED,
100 State.SESSION_INITIALIZED_PRE_APPROVED,
101 State.TERMINATED_SUCCESS,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
105 ));
106 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
107 State.SESSION_ACCEPTED,
108 State.TERMINATED_SUCCESS,
109 State.TERMINATED_DECLINED_OR_BUSY,
110 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
111 State.TERMINATED_CANCEL_OR_TIMEOUT,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
125 State.TERMINATED_SUCCESS,
126 State.TERMINATED_DECLINED_OR_BUSY,
127 State.TERMINATED_CONNECTIVITY_ERROR,
128 State.TERMINATED_CANCEL_OR_TIMEOUT,
129 State.TERMINATED_APPLICATION_FAILURE,
130 State.TERMINATED_SECURITY_ERROR
131 ));
132 VALID_TRANSITIONS = transitionBuilder.build();
133 }
134
135 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
136 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
137 private final OmemoVerification omemoVerification = new OmemoVerification();
138 private final Message message;
139 private State state = State.NULL;
140 private StateTransitionException stateTransitionException;
141 private Set<Media> proposedMedia;
142 private RtpContentMap initiatorRtpContentMap;
143 private RtpContentMap responderRtpContentMap;
144 private long rtpConnectionStarted = 0; //time of 'connected'
145 private long rtpConnectionEnded = 0;
146 private ScheduledFuture<?> ringingTimeoutFuture;
147
148 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
149 super(jingleConnectionManager, id, initiator);
150 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
151 id.account,
152 id.with.asBareJid(),
153 false,
154 false
155 );
156 this.message = new Message(
157 conversation,
158 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
159 Message.TYPE_RTP_SESSION,
160 id.sessionId
161 );
162 }
163
164 private static State reasonToState(Reason reason) {
165 switch (reason) {
166 case SUCCESS:
167 return State.TERMINATED_SUCCESS;
168 case DECLINE:
169 case BUSY:
170 return State.TERMINATED_DECLINED_OR_BUSY;
171 case CANCEL:
172 case TIMEOUT:
173 return State.TERMINATED_CANCEL_OR_TIMEOUT;
174 case SECURITY_ERROR:
175 return State.TERMINATED_SECURITY_ERROR;
176 case FAILED_APPLICATION:
177 case UNSUPPORTED_TRANSPORTS:
178 case UNSUPPORTED_APPLICATIONS:
179 return State.TERMINATED_APPLICATION_FAILURE;
180 default:
181 return State.TERMINATED_CONNECTIVITY_ERROR;
182 }
183 }
184
185 @Override
186 synchronized void deliverPacket(final JinglePacket jinglePacket) {
187 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
188 switch (jinglePacket.getAction()) {
189 case SESSION_INITIATE:
190 receiveSessionInitiate(jinglePacket);
191 break;
192 case TRANSPORT_INFO:
193 receiveTransportInfo(jinglePacket);
194 break;
195 case SESSION_ACCEPT:
196 receiveSessionAccept(jinglePacket);
197 break;
198 case SESSION_TERMINATE:
199 receiveSessionTerminate(jinglePacket);
200 break;
201 default:
202 respondOk(jinglePacket);
203 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
204 break;
205 }
206 }
207
208 @Override
209 synchronized void notifyRebound() {
210 if (isTerminated()) {
211 return;
212 }
213 webRTCWrapper.close();
214 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
215 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
216 }
217 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
218 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
219 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
220 } else {
221 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
222 finish();
223 }
224 }
225
226 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
227 respondOk(jinglePacket);
228 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
229 final State previous = this.state;
230 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
231 if (TERMINATED.contains(previous)) {
232 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
233 return;
234 }
235 webRTCWrapper.close();
236 final State target = reasonToState(wrapper.reason);
237 transitionOrThrow(target);
238 writeLogMessage(target);
239 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
240 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
241 }
242 finish();
243 }
244
245 private void receiveTransportInfo(final JinglePacket jinglePacket) {
246 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
247 respondOk(jinglePacket);
248 final RtpContentMap contentMap;
249 try {
250 contentMap = RtpContentMap.of(jinglePacket);
251 } catch (IllegalArgumentException | NullPointerException e) {
252 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
253 return;
254 }
255 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
256 if (this.state == State.SESSION_ACCEPTED) {
257 try {
258 processCandidates(candidates);
259 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
260 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
261 }
262 } else {
263 pendingIceCandidates.push(candidates);
264 }
265 } else {
266 if (isTerminated()) {
267 respondOk(jinglePacket);
268 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
269 } else {
270 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
271 terminateWithOutOfOrder(jinglePacket);
272 }
273 }
274 }
275
276 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
277 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
278 final Group originalGroup = rtpContentMap.group;
279 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
280 if (identificationTags.size() == 0) {
281 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
282 }
283 processCandidates(identificationTags, contents);
284 }
285
286 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
287 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
288 final String ufrag = content.getValue().transport.getAttribute("ufrag");
289 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
290 final String sdp;
291 try {
292 sdp = candidate.toSdpAttribute(ufrag);
293 } catch (IllegalArgumentException e) {
294 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
295 continue;
296 }
297 final String sdpMid = content.getKey();
298 final int mLineIndex = indices.indexOf(sdpMid);
299 if (mLineIndex < 0) {
300 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
301 }
302 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
303 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
304 this.webRTCWrapper.addIceCandidate(iceCandidate);
305 }
306 }
307 }
308
309 private RtpContentMap receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
310 final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket);
311 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
312 final AxolotlService.OmemoVerifiedPayload<RtpContentMap> omemoVerifiedPayload;
313 try {
314 omemoVerifiedPayload = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
315 } catch (final CryptoFailedException e) {
316 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
317 }
318 this.omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
319 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + this.omemoVerification);
320 return omemoVerifiedPayload.getPayload();
321 } else if (expectVerification) {
322 throw new SecurityException("DTLS fingerprint was unexpectedly not verifiable");
323 } else {
324 return receivedContentMap;
325 }
326 }
327
328 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
329 if (isInitiator()) {
330 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
331 if (isTerminated()) {
332 Log.d(Config.LOGTAG, String.format(
333 "%s: got a reason to terminate with out-of-order. but already in state %s",
334 id.account.getJid().asBareJid(),
335 getState()
336 ));
337 respondWithOutOfOrder(jinglePacket);
338 } else {
339 terminateWithOutOfOrder(jinglePacket);
340 }
341 return;
342 }
343 final RtpContentMap contentMap;
344 try {
345 contentMap = receiveRtpContentMap(jinglePacket, false);
346 contentMap.requireContentDescriptions();
347 contentMap.requireDTLSFingerprint();
348 } catch (final RuntimeException e) {
349 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
350 respondOk(jinglePacket);
351 sendSessionTerminate(Reason.of(e), e.getMessage());
352 return;
353 }
354 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
355 final State target;
356 if (this.state == State.PROCEED) {
357 Preconditions.checkState(
358 proposedMedia != null && proposedMedia.size() > 0,
359 "proposed media must be set when processing pre-approved session-initiate"
360 );
361 if (!this.proposedMedia.equals(contentMap.getMedia())) {
362 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
363 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
364 this.proposedMedia,
365 contentMap.getMedia()
366 ));
367 return;
368 }
369 target = State.SESSION_INITIALIZED_PRE_APPROVED;
370 } else {
371 target = State.SESSION_INITIALIZED;
372 }
373 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
374 respondOk(jinglePacket);
375
376 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
377 if (candidates.size() > 0) {
378 pendingIceCandidates.push(candidates);
379 }
380 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
381 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
382 sendSessionAccept();
383 } else {
384 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
385 startRinging();
386 }
387 } else {
388 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
389 terminateWithOutOfOrder(jinglePacket);
390 }
391 }
392
393 private void receiveSessionAccept(final JinglePacket jinglePacket) {
394 if (!isInitiator()) {
395 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
396 terminateWithOutOfOrder(jinglePacket);
397 return;
398 }
399 final RtpContentMap contentMap;
400 try {
401 contentMap = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
402 contentMap.requireContentDescriptions();
403 contentMap.requireDTLSFingerprint();
404 } catch (final RuntimeException e) {
405 respondOk(jinglePacket);
406 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
407 webRTCWrapper.close();
408 sendSessionTerminate(Reason.of(e), e.getMessage());
409 return;
410 }
411 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
412 if (!initiatorMedia.equals(contentMap.getMedia())) {
413 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
414 "Your session-included included media %s but our session-initiate was %s",
415 this.proposedMedia,
416 contentMap.getMedia()
417 ));
418 return;
419 }
420 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
421 if (transition(State.SESSION_ACCEPTED)) {
422 respondOk(jinglePacket);
423 receiveSessionAccept(contentMap);
424 } else {
425 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
426 respondOk(jinglePacket);
427 }
428 }
429
430 private void receiveSessionAccept(final RtpContentMap contentMap) {
431 this.responderRtpContentMap = contentMap;
432 final SessionDescription sessionDescription;
433 try {
434 sessionDescription = SessionDescription.of(contentMap);
435 } catch (final IllegalArgumentException | NullPointerException e) {
436 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
437 webRTCWrapper.close();
438 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
439 return;
440 }
441 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
442 org.webrtc.SessionDescription.Type.ANSWER,
443 sessionDescription.toString()
444 );
445 try {
446 this.webRTCWrapper.setRemoteDescription(answer).get();
447 } catch (final Exception e) {
448 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
449 webRTCWrapper.close();
450 sendSessionTerminate(Reason.FAILED_APPLICATION);
451 return;
452 }
453 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
454 processCandidates(identificationTags, contentMap.contents.entrySet());
455 }
456
457 private void sendSessionAccept() {
458 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
459 if (rtpContentMap == null) {
460 throw new IllegalStateException("initiator RTP Content Map has not been set");
461 }
462 final SessionDescription offer;
463 try {
464 offer = SessionDescription.of(rtpContentMap);
465 } catch (final IllegalArgumentException | NullPointerException e) {
466 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
467 webRTCWrapper.close();
468 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
469 return;
470 }
471 sendSessionAccept(rtpContentMap.getMedia(), offer);
472 }
473
474 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
475 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
476 }
477
478 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
479 if (isTerminated()) {
480 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
481 return;
482 }
483 try {
484 setupWebRTC(media, iceServers);
485 } catch (final WebRTCWrapper.InitializationException e) {
486 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
487 webRTCWrapper.close();
488 sendSessionTerminate(Reason.FAILED_APPLICATION);
489 return;
490 }
491 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
492 org.webrtc.SessionDescription.Type.OFFER,
493 offer.toString()
494 );
495 try {
496 this.webRTCWrapper.setRemoteDescription(sdp).get();
497 addIceCandidatesFromBlackLog();
498 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
499 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
500 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
501 sendSessionAccept(respondingRtpContentMap);
502 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
503 } catch (final Exception e) {
504 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(e));
505 webRTCWrapper.close();
506 sendSessionTerminate(Reason.FAILED_APPLICATION);
507 }
508 }
509
510 private void addIceCandidatesFromBlackLog() {
511 while (!this.pendingIceCandidates.isEmpty()) {
512 processCandidates(this.pendingIceCandidates.poll());
513 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
514 }
515 }
516
517 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
518 this.responderRtpContentMap = rtpContentMap;
519 this.transitionOrThrow(State.SESSION_ACCEPTED);
520 final RtpContentMap outgoingContentMap;
521 if (this.omemoVerification.hasDeviceId()) {
522 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
523 try {
524 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
525 outgoingContentMap = verifiedPayload.getPayload();
526 this.omemoVerification.setOrEnsureEqual(verifiedPayload);
527 } catch (final Exception e) {
528 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
529 }
530 } else {
531 outgoingContentMap = rtpContentMap;
532 }
533 final JinglePacket sessionAccept = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
534 send(sessionAccept);
535 }
536
537 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
538 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
539 switch (message.getName()) {
540 case "propose":
541 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
542 break;
543 case "proceed":
544 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
545 break;
546 case "retract":
547 receiveRetract(from, serverMessageId, timestamp);
548 break;
549 case "reject":
550 receiveReject(from, serverMessageId, timestamp);
551 break;
552 case "accept":
553 receiveAccept(from, serverMessageId, timestamp);
554 break;
555 default:
556 break;
557 }
558 }
559
560 void deliverFailedProceed() {
561 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
562 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
563 webRTCWrapper.close();
564 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
565 this.finish();
566 }
567 }
568
569 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
570 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
571 if (originatedFromMyself) {
572 if (transition(State.ACCEPTED)) {
573 if (serverMsgId != null) {
574 this.message.setServerMsgId(serverMsgId);
575 }
576 this.message.setTime(timestamp);
577 this.message.setCarbon(true); //indicate that call was accepted on other device
578 this.writeLogMessageSuccess(0);
579 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
580 this.finish();
581 } else {
582 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
583 }
584 } else {
585 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
586 }
587 }
588
589 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
590 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
591 //reject from another one of my clients
592 if (originatedFromMyself) {
593 receiveRejectFromMyself(serverMsgId, timestamp);
594 } else if (isInitiator()) {
595 if (from.equals(id.with)) {
596 receiveRejectFromResponder();
597 } else {
598 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
599 }
600 } else {
601 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
602 }
603 }
604
605 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
606 if (transition(State.REJECTED)) {
607 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
608 this.finish();
609 if (serverMsgId != null) {
610 this.message.setServerMsgId(serverMsgId);
611 }
612 this.message.setTime(timestamp);
613 this.message.setCarbon(true); //indicate that call was rejected on other device
614 writeLogMessageMissed();
615 } else {
616 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
617 }
618 }
619
620 private void receiveRejectFromResponder() {
621 if (isInState(State.PROCEED)) {
622 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
623 closeTransitionLogFinish(State.REJECTED_RACED);
624 return;
625 }
626 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
627 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
628 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
629 return;
630 }
631 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
632 }
633
634 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
635 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
636 if (originatedFromMyself) {
637 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
638 } else if (transition(State.PROPOSED, () -> {
639 final Collection<RtpDescription> descriptions = Collections2.transform(
640 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
641 input -> (RtpDescription) input
642 );
643 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
644 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
645 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
646 this.proposedMedia = Sets.newHashSet(media);
647 })) {
648 if (serverMsgId != null) {
649 this.message.setServerMsgId(serverMsgId);
650 }
651 this.message.setTime(timestamp);
652 startRinging();
653 } else {
654 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
655 }
656 }
657
658 private void startRinging() {
659 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
660 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
661 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
662 }
663
664 private synchronized void ringingTimeout() {
665 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
666 switch (this.state) {
667 case PROPOSED:
668 message.markUnread();
669 rejectCallFromProposed();
670 break;
671 case SESSION_INITIALIZED:
672 message.markUnread();
673 rejectCallFromSessionInitiate();
674 break;
675 }
676 }
677
678 private void cancelRingingTimeout() {
679 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
680 if (future != null && !future.isCancelled()) {
681 future.cancel(false);
682 }
683 }
684
685 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
686 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
687 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
688 if (from.equals(id.with)) {
689 if (isInitiator()) {
690 if (transition(State.PROCEED)) {
691 if (serverMsgId != null) {
692 this.message.setServerMsgId(serverMsgId);
693 }
694 this.message.setTime(timestamp);
695 final Integer remoteDeviceId = proceed.getDeviceId();
696 if (isOmemoEnabled()) {
697 this.omemoVerification.setDeviceId(remoteDeviceId);
698 } else {
699 if (remoteDeviceId != null) {
700 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
701 }
702 this.omemoVerification.setDeviceId(null);
703 }
704 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
705 } else {
706 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
707 }
708 } else {
709 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
710 }
711 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
712 if (transition(State.ACCEPTED)) {
713 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
714 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
715 this.finish();
716 }
717 } else {
718 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
719 }
720 }
721
722 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
723 if (from.equals(id.with)) {
724 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
725 if (transition(target)) {
726 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
727 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
728 if (serverMsgId != null) {
729 this.message.setServerMsgId(serverMsgId);
730 }
731 this.message.setTime(timestamp);
732 if (target == State.RETRACTED) {
733 this.message.markUnread();
734 }
735 writeLogMessageMissed();
736 finish();
737 } else {
738 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
739 }
740 } else {
741 //TODO parse retract from self
742 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
743 }
744 }
745
746 public void sendSessionInitiate() {
747 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
748 }
749
750 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
751 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
752 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
753 }
754
755 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
756 if (isTerminated()) {
757 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
758 return;
759 }
760 try {
761 setupWebRTC(media, iceServers);
762 } catch (final WebRTCWrapper.InitializationException e) {
763 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
764 webRTCWrapper.close();
765 sendRetract(Reason.ofException(e));
766 return;
767 }
768 try {
769 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
770 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
771 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
772 sendSessionInitiate(rtpContentMap, targetState);
773 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
774 } catch (final Exception e) {
775 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(e));
776 webRTCWrapper.close();
777 final Reason reason = Reason.ofException(e);
778 if (isInState(targetState)) {
779 sendSessionTerminate(reason);
780 } else {
781 sendRetract(reason);
782 }
783 }
784 }
785
786 private void sendRetract(final Reason reason) {
787 //TODO embed reason into retract
788 sendJingleMessage("retract", id.with.asBareJid());
789 transitionOrThrow(reasonToState(reason));
790 this.finish();
791 }
792
793 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
794 this.initiatorRtpContentMap = rtpContentMap;
795 final RtpContentMap outgoingContentMap = encryptSessionInitiate(rtpContentMap);
796 final JinglePacket sessionInitiate = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
797 this.transitionOrThrow(targetState);
798 send(sessionInitiate);
799 }
800
801 private RtpContentMap encryptSessionInitiate(final RtpContentMap rtpContentMap) {
802 if (this.omemoVerification.hasDeviceId()) {
803 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
804 try {
805 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
806 } catch (final CryptoFailedException e) {
807 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
808 return rtpContentMap;
809 }
810 this.omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
811 return verifiedPayload.getPayload();
812 } else {
813 return rtpContentMap;
814 }
815 }
816
817 private void sendSessionTerminate(final Reason reason) {
818 sendSessionTerminate(reason, null);
819 }
820
821 private void sendSessionTerminate(final Reason reason, final String text) {
822 final State previous = this.state;
823 final State target = reasonToState(reason);
824 transitionOrThrow(target);
825 if (previous != State.NULL) {
826 writeLogMessage(target);
827 }
828 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
829 jinglePacket.setReason(reason, text);
830 Log.d(Config.LOGTAG, jinglePacket.toString());
831 send(jinglePacket);
832 finish();
833 }
834
835 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
836 final RtpContentMap transportInfo;
837 try {
838 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
839 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
840 } catch (final Exception e) {
841 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
842 return;
843 }
844 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
845 send(jinglePacket);
846 }
847
848 private void send(final JinglePacket jinglePacket) {
849 jinglePacket.setTo(id.with);
850 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
851 }
852
853 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
854 if (response.getType() == IqPacket.TYPE.ERROR) {
855 final String errorCondition = response.getErrorCondition();
856 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
857 if (isTerminated()) {
858 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
859 return;
860 }
861 this.webRTCWrapper.close();
862 final State target;
863 if (Arrays.asList(
864 "service-unavailable",
865 "recipient-unavailable",
866 "remote-server-not-found",
867 "remote-server-timeout"
868 ).contains(errorCondition)) {
869 target = State.TERMINATED_CONNECTIVITY_ERROR;
870 } else {
871 target = State.TERMINATED_APPLICATION_FAILURE;
872 }
873 transitionOrThrow(target);
874 this.finish();
875 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
876 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
877 if (isTerminated()) {
878 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
879 return;
880 }
881 this.webRTCWrapper.close();
882 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
883 this.finish();
884 }
885 }
886
887 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
888 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
889 this.webRTCWrapper.close();
890 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
891 respondWithOutOfOrder(jinglePacket);
892 this.finish();
893 }
894
895 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
896 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
897 }
898
899 private void respondOk(final JinglePacket jinglePacket) {
900 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
901 }
902
903 public void throwStateTransitionException() {
904 final StateTransitionException exception = this.stateTransitionException;
905 if (exception != null) {
906 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
907 }
908 }
909
910 public RtpEndUserState getEndUserState() {
911 switch (this.state) {
912 case NULL:
913 case PROPOSED:
914 case SESSION_INITIALIZED:
915 if (isInitiator()) {
916 return RtpEndUserState.RINGING;
917 } else {
918 return RtpEndUserState.INCOMING_CALL;
919 }
920 case PROCEED:
921 if (isInitiator()) {
922 return RtpEndUserState.RINGING;
923 } else {
924 return RtpEndUserState.ACCEPTING_CALL;
925 }
926 case SESSION_INITIALIZED_PRE_APPROVED:
927 if (isInitiator()) {
928 return RtpEndUserState.RINGING;
929 } else {
930 return RtpEndUserState.CONNECTING;
931 }
932 case SESSION_ACCEPTED:
933 final PeerConnection.PeerConnectionState state;
934 try {
935 state = webRTCWrapper.getState();
936 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
937 //We usually close the WebRTCWrapper *before* transitioning so we might still
938 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
939 return RtpEndUserState.ENDING_CALL;
940 }
941 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
942 return RtpEndUserState.CONNECTED;
943 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
944 return RtpEndUserState.CONNECTING;
945 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
946 return RtpEndUserState.ENDING_CALL;
947 } else {
948 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
949 }
950 case REJECTED:
951 case REJECTED_RACED:
952 case TERMINATED_DECLINED_OR_BUSY:
953 if (isInitiator()) {
954 return RtpEndUserState.DECLINED_OR_BUSY;
955 } else {
956 return RtpEndUserState.ENDED;
957 }
958 case TERMINATED_SUCCESS:
959 case ACCEPTED:
960 case RETRACTED:
961 case TERMINATED_CANCEL_OR_TIMEOUT:
962 return RtpEndUserState.ENDED;
963 case RETRACTED_RACED:
964 if (isInitiator()) {
965 return RtpEndUserState.ENDED;
966 } else {
967 return RtpEndUserState.RETRACTED;
968 }
969 case TERMINATED_CONNECTIVITY_ERROR:
970 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
971 case TERMINATED_APPLICATION_FAILURE:
972 return RtpEndUserState.APPLICATION_ERROR;
973 case TERMINATED_SECURITY_ERROR:
974 return RtpEndUserState.SECURITY_ERROR;
975 }
976 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
977 }
978
979 public Set<Media> getMedia() {
980 final State current = getState();
981 if (current == State.NULL) {
982 if (isInitiator()) {
983 return Preconditions.checkNotNull(
984 this.proposedMedia,
985 "RTP connection has not been initialized properly"
986 );
987 }
988 throw new IllegalStateException("RTP connection has not been initialized yet");
989 }
990 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
991 return Preconditions.checkNotNull(
992 this.proposedMedia,
993 "RTP connection has not been initialized properly"
994 );
995 }
996 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
997 if (initiatorContentMap != null) {
998 return initiatorContentMap.getMedia();
999 } else if (isTerminated()) {
1000 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1001 } else {
1002 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1003 }
1004 }
1005
1006
1007 public boolean isVerified() {
1008 final String fingerprint = this.omemoVerification.getFingerprint();
1009 if (fingerprint == null) {
1010 return false;
1011 }
1012 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1013 return status != null && status.getTrust() == FingerprintStatus.Trust.VERIFIED;
1014 }
1015
1016 public synchronized void acceptCall() {
1017 switch (this.state) {
1018 case PROPOSED:
1019 cancelRingingTimeout();
1020 acceptCallFromProposed();
1021 break;
1022 case SESSION_INITIALIZED:
1023 cancelRingingTimeout();
1024 acceptCallFromSessionInitialized();
1025 break;
1026 case ACCEPTED:
1027 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1028 break;
1029 case PROCEED:
1030 case SESSION_ACCEPTED:
1031 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1032 break;
1033 default:
1034 throw new IllegalStateException("Can not accept call from " + this.state);
1035 }
1036 }
1037
1038
1039 public void notifyPhoneCall() {
1040 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1041 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1042 rejectCall();
1043 } else {
1044 endCall();
1045 }
1046 }
1047
1048 public synchronized void rejectCall() {
1049 if (isTerminated()) {
1050 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1051 return;
1052 }
1053 switch (this.state) {
1054 case PROPOSED:
1055 rejectCallFromProposed();
1056 break;
1057 case SESSION_INITIALIZED:
1058 rejectCallFromSessionInitiate();
1059 break;
1060 default:
1061 throw new IllegalStateException("Can not reject call from " + this.state);
1062 }
1063 }
1064
1065 public synchronized void endCall() {
1066 if (isTerminated()) {
1067 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1068 return;
1069 }
1070 if (isInState(State.PROPOSED) && !isInitiator()) {
1071 rejectCallFromProposed();
1072 return;
1073 }
1074 if (isInState(State.PROCEED)) {
1075 if (isInitiator()) {
1076 retractFromProceed();
1077 } else {
1078 rejectCallFromProceed();
1079 }
1080 return;
1081 }
1082 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1083 this.webRTCWrapper.close();
1084 sendSessionTerminate(Reason.CANCEL);
1085 return;
1086 }
1087 if (isInState(State.SESSION_INITIALIZED)) {
1088 rejectCallFromSessionInitiate();
1089 return;
1090 }
1091 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1092 this.webRTCWrapper.close();
1093 sendSessionTerminate(Reason.SUCCESS);
1094 return;
1095 }
1096 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1097 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1098 return;
1099 }
1100 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1101 }
1102
1103 private void retractFromProceed() {
1104 Log.d(Config.LOGTAG, "retract from proceed");
1105 this.sendJingleMessage("retract");
1106 closeTransitionLogFinish(State.RETRACTED_RACED);
1107 }
1108
1109 private void closeTransitionLogFinish(final State state) {
1110 this.webRTCWrapper.close();
1111 transitionOrThrow(state);
1112 writeLogMessage(state);
1113 finish();
1114 }
1115
1116 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1117 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1118 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1119 if (media.contains(Media.VIDEO)) {
1120 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1121 } else {
1122 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1123 }
1124 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1125 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1126 }
1127
1128 private void acceptCallFromProposed() {
1129 transitionOrThrow(State.PROCEED);
1130 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1131 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1132 this.sendJingleMessage("proceed");
1133 }
1134
1135 private void rejectCallFromProposed() {
1136 transitionOrThrow(State.REJECTED);
1137 writeLogMessageMissed();
1138 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1139 this.sendJingleMessage("reject");
1140 finish();
1141 }
1142
1143 private void rejectCallFromProceed() {
1144 this.sendJingleMessage("reject");
1145 closeTransitionLogFinish(State.REJECTED_RACED);
1146 }
1147
1148 private void rejectCallFromSessionInitiate() {
1149 webRTCWrapper.close();
1150 sendSessionTerminate(Reason.DECLINE);
1151 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1152 }
1153
1154 private void sendJingleMessage(final String action) {
1155 sendJingleMessage(action, id.with);
1156 }
1157
1158 private void sendJingleMessage(final String action, final Jid to) {
1159 final MessagePacket messagePacket = new MessagePacket();
1160 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1161 messagePacket.setTo(to);
1162 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1163 if ("proceed".equals(action)) {
1164 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1165 if (isOmemoEnabled()) {
1166 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1167 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1168 device.setAttribute("id", deviceId);
1169 }
1170 }
1171 messagePacket.addChild("store", "urn:xmpp:hints");
1172 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1173 }
1174
1175 private boolean isOmemoEnabled() {
1176 final Conversational conversational = message.getConversation();
1177 if (conversational instanceof Conversation) {
1178 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1179 }
1180 return false;
1181 }
1182
1183 private void acceptCallFromSessionInitialized() {
1184 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1185 sendSessionAccept();
1186 }
1187
1188 private synchronized boolean isInState(State... state) {
1189 return Arrays.asList(state).contains(this.state);
1190 }
1191
1192 private boolean transition(final State target) {
1193 return transition(target, null);
1194 }
1195
1196 private synchronized boolean transition(final State target, final Runnable runnable) {
1197 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1198 if (validTransitions != null && validTransitions.contains(target)) {
1199 this.state = target;
1200 this.stateTransitionException = new StateTransitionException(target);
1201 if (runnable != null) {
1202 runnable.run();
1203 }
1204 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1205 updateEndUserState();
1206 updateOngoingCallNotification();
1207 return true;
1208 } else {
1209 return false;
1210 }
1211 }
1212
1213 void transitionOrThrow(final State target) {
1214 if (!transition(target)) {
1215 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1216 }
1217 }
1218
1219 @Override
1220 public void onIceCandidate(final IceCandidate iceCandidate) {
1221 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1222 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1223 sendTransportInfo(iceCandidate.sdpMid, candidate);
1224 }
1225
1226 @Override
1227 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1228 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1229 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
1230 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
1231 }
1232 if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) {
1233 this.rtpConnectionEnded = SystemClock.elapsedRealtime();
1234 }
1235 //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace
1236 //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable
1237 //as there is no content-replace
1238 if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) {
1239 if (isTerminated()) {
1240 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1241 return;
1242 }
1243 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1244 } else {
1245 updateEndUserState();
1246 }
1247 }
1248
1249 private void closeWebRTCSessionAfterFailedConnection() {
1250 this.webRTCWrapper.close();
1251 synchronized (this) {
1252 if (isTerminated()) {
1253 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1254 return;
1255 }
1256 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1257 }
1258 }
1259
1260 public long getRtpConnectionStarted() {
1261 return this.rtpConnectionStarted;
1262 }
1263
1264 public long getRtpConnectionEnded() {
1265 return this.rtpConnectionEnded;
1266 }
1267
1268 public AppRTCAudioManager getAudioManager() {
1269 return webRTCWrapper.getAudioManager();
1270 }
1271
1272 public boolean isMicrophoneEnabled() {
1273 return webRTCWrapper.isMicrophoneEnabled();
1274 }
1275
1276 public boolean setMicrophoneEnabled(final boolean enabled) {
1277 return webRTCWrapper.setMicrophoneEnabled(enabled);
1278 }
1279
1280 public boolean isVideoEnabled() {
1281 return webRTCWrapper.isVideoEnabled();
1282 }
1283
1284 public void setVideoEnabled(final boolean enabled) {
1285 webRTCWrapper.setVideoEnabled(enabled);
1286 }
1287
1288 public boolean isCameraSwitchable() {
1289 return webRTCWrapper.isCameraSwitchable();
1290 }
1291
1292 public boolean isFrontCamera() {
1293 return webRTCWrapper.isFrontCamera();
1294 }
1295
1296 public ListenableFuture<Boolean> switchCamera() {
1297 return webRTCWrapper.switchCamera();
1298 }
1299
1300 @Override
1301 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1302 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1303 }
1304
1305 private void updateEndUserState() {
1306 final RtpEndUserState endUserState = getEndUserState();
1307 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1308 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1309 }
1310
1311 private void updateOngoingCallNotification() {
1312 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1313 xmppConnectionService.setOngoingCall(id, getMedia());
1314 } else {
1315 xmppConnectionService.removeOngoingCall();
1316 }
1317 }
1318
1319 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1320 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1321 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1322 request.setTo(id.account.getDomain());
1323 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1324 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1325 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1326 if (response.getType() == IqPacket.TYPE.RESULT) {
1327 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1328 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1329 for (final Element child : children) {
1330 if ("service".equals(child.getName())) {
1331 final String type = child.getAttribute("type");
1332 final String host = child.getAttribute("host");
1333 final String sport = child.getAttribute("port");
1334 final Integer port = sport == null ? null : Ints.tryParse(sport);
1335 final String transport = child.getAttribute("transport");
1336 final String username = child.getAttribute("username");
1337 final String password = child.getAttribute("password");
1338 if (Strings.isNullOrEmpty(host) || port == null) {
1339 continue;
1340 }
1341 if (port < 0 || port > 65535) {
1342 continue;
1343 }
1344 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1345 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1346 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1347 continue;
1348 }
1349 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1350 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1351 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1352 if (username != null && password != null) {
1353 iceServerBuilder.setUsername(username);
1354 iceServerBuilder.setPassword(password);
1355 } else if (Arrays.asList("turn", "turns").contains(type)) {
1356 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1357 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1358 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1359 continue;
1360 }
1361 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1362 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1363 listBuilder.add(iceServer);
1364 }
1365 }
1366 }
1367 }
1368 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1369 if (iceServers.size() == 0) {
1370 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1371 }
1372 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1373 });
1374 } else {
1375 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1376 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1377 }
1378 }
1379
1380 private void finish() {
1381 if (isTerminated()) {
1382 this.cancelRingingTimeout();
1383 this.webRTCWrapper.verifyClosed();
1384 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1385 this.jingleConnectionManager.finishConnectionOrThrow(this);
1386 } else {
1387 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1388 }
1389 }
1390
1391 private void writeLogMessage(final State state) {
1392 final long started = this.rtpConnectionStarted;
1393 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
1394 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1395 writeLogMessageSuccess(duration);
1396 } else {
1397 writeLogMessageMissed();
1398 }
1399 }
1400
1401 private void writeLogMessageSuccess(final long duration) {
1402 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1403 this.writeMessage();
1404 }
1405
1406 private void writeLogMessageMissed() {
1407 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1408 this.writeMessage();
1409 }
1410
1411 private void writeMessage() {
1412 final Conversational conversational = message.getConversation();
1413 if (conversational instanceof Conversation) {
1414 ((Conversation) conversational).add(this.message);
1415 xmppConnectionService.createMessageAsync(message);
1416 xmppConnectionService.updateConversationUi();
1417 } else {
1418 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1419 }
1420 }
1421
1422 public State getState() {
1423 return this.state;
1424 }
1425
1426 boolean isTerminated() {
1427 return TERMINATED.contains(this.state);
1428 }
1429
1430 public Optional<VideoTrack> getLocalVideoTrack() {
1431 return webRTCWrapper.getLocalVideoTrack();
1432 }
1433
1434 public Optional<VideoTrack> getRemoteVideoTrack() {
1435 return webRTCWrapper.getRemoteVideoTrack();
1436 }
1437
1438
1439 public EglBase.Context getEglBaseContext() {
1440 return webRTCWrapper.getEglBaseContext();
1441 }
1442
1443 void setProposedMedia(final Set<Media> media) {
1444 this.proposedMedia = media;
1445 }
1446
1447 public void fireStateUpdate() {
1448 final RtpEndUserState endUserState = getEndUserState();
1449 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1450 }
1451
1452 private interface OnIceServersDiscovered {
1453 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1454 }
1455
1456 private static class StateTransitionException extends Exception {
1457 private final State state;
1458
1459 private StateTransitionException(final State state) {
1460 this.state = state;
1461 }
1462 }
1463}