1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import com.google.common.base.Optional;
7import com.google.common.base.Preconditions;
8import com.google.common.base.Strings;
9import com.google.common.base.Throwables;
10import com.google.common.collect.Collections2;
11import com.google.common.collect.ImmutableList;
12import com.google.common.collect.ImmutableMap;
13import com.google.common.collect.Sets;
14import com.google.common.primitives.Ints;
15import com.google.common.util.concurrent.ListenableFuture;
16
17import org.webrtc.EglBase;
18import org.webrtc.IceCandidate;
19import org.webrtc.PeerConnection;
20import org.webrtc.VideoTrack;
21
22import java.util.ArrayDeque;
23import java.util.Arrays;
24import java.util.Collection;
25import java.util.Collections;
26import java.util.List;
27import java.util.Map;
28import java.util.Set;
29import java.util.concurrent.ScheduledFuture;
30import java.util.concurrent.TimeUnit;
31
32import eu.siacs.conversations.Config;
33import eu.siacs.conversations.crypto.axolotl.AxolotlService;
34import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
35import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
36import eu.siacs.conversations.entities.Account;
37import eu.siacs.conversations.entities.Conversation;
38import eu.siacs.conversations.entities.Conversational;
39import eu.siacs.conversations.entities.Message;
40import eu.siacs.conversations.entities.RtpSessionStatus;
41import eu.siacs.conversations.services.AppRTCAudioManager;
42import eu.siacs.conversations.utils.IP;
43import eu.siacs.conversations.xml.Element;
44import eu.siacs.conversations.xml.Namespace;
45import eu.siacs.conversations.xmpp.Jid;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
57
58 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
59 State.PROCEED,
60 State.SESSION_INITIALIZED_PRE_APPROVED,
61 State.SESSION_ACCEPTED
62 );
63 private static final long BUSY_TIME_OUT = 30;
64 private static final List<State> TERMINATED = Arrays.asList(
65 State.ACCEPTED,
66 State.REJECTED,
67 State.REJECTED_RACED,
68 State.RETRACTED,
69 State.RETRACTED_RACED,
70 State.TERMINATED_SUCCESS,
71 State.TERMINATED_DECLINED_OR_BUSY,
72 State.TERMINATED_CONNECTIVITY_ERROR,
73 State.TERMINATED_CANCEL_OR_TIMEOUT,
74 State.TERMINATED_APPLICATION_FAILURE
75 );
76
77 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
78
79 static {
80 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
81 transitionBuilder.put(State.NULL, ImmutableList.of(
82 State.PROPOSED,
83 State.SESSION_INITIALIZED,
84 State.TERMINATED_APPLICATION_FAILURE
85 ));
86 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
87 State.ACCEPTED,
88 State.PROCEED,
89 State.REJECTED,
90 State.RETRACTED,
91 State.TERMINATED_APPLICATION_FAILURE,
92 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
93 ));
94 transitionBuilder.put(State.PROCEED, ImmutableList.of(
95 State.REJECTED_RACED,
96 State.RETRACTED_RACED,
97 State.SESSION_INITIALIZED_PRE_APPROVED,
98 State.TERMINATED_SUCCESS,
99 State.TERMINATED_APPLICATION_FAILURE,
100 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
101 ));
102 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
103 State.SESSION_ACCEPTED,
104 State.TERMINATED_SUCCESS,
105 State.TERMINATED_DECLINED_OR_BUSY,
106 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
107 State.TERMINATED_CANCEL_OR_TIMEOUT,
108 State.TERMINATED_APPLICATION_FAILURE
109 ));
110 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
111 State.SESSION_ACCEPTED,
112 State.TERMINATED_SUCCESS,
113 State.TERMINATED_DECLINED_OR_BUSY,
114 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
115 State.TERMINATED_CANCEL_OR_TIMEOUT,
116 State.TERMINATED_APPLICATION_FAILURE
117 ));
118 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
119 State.TERMINATED_SUCCESS,
120 State.TERMINATED_DECLINED_OR_BUSY,
121 State.TERMINATED_CONNECTIVITY_ERROR,
122 State.TERMINATED_CANCEL_OR_TIMEOUT,
123 State.TERMINATED_APPLICATION_FAILURE
124 ));
125 VALID_TRANSITIONS = transitionBuilder.build();
126 }
127
128 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
129 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
130 private final OmemoVerification omemoVerification = new OmemoVerification();
131 private final Message message;
132 private State state = State.NULL;
133 private StateTransitionException stateTransitionException;
134 private Set<Media> proposedMedia;
135 private RtpContentMap initiatorRtpContentMap;
136 private RtpContentMap responderRtpContentMap;
137 private long rtpConnectionStarted = 0; //time of 'connected'
138 private long rtpConnectionEnded = 0;
139 private ScheduledFuture<?> ringingTimeoutFuture;
140
141 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
142 super(jingleConnectionManager, id, initiator);
143 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
144 id.account,
145 id.with.asBareJid(),
146 false,
147 false
148 );
149 this.message = new Message(
150 conversation,
151 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
152 Message.TYPE_RTP_SESSION,
153 id.sessionId
154 );
155 }
156
157 private static State reasonToState(Reason reason) {
158 switch (reason) {
159 case SUCCESS:
160 return State.TERMINATED_SUCCESS;
161 case DECLINE:
162 case BUSY:
163 return State.TERMINATED_DECLINED_OR_BUSY;
164 case CANCEL:
165 case TIMEOUT:
166 return State.TERMINATED_CANCEL_OR_TIMEOUT;
167 case FAILED_APPLICATION:
168 case SECURITY_ERROR:
169 case UNSUPPORTED_TRANSPORTS:
170 case UNSUPPORTED_APPLICATIONS:
171 return State.TERMINATED_APPLICATION_FAILURE;
172 default:
173 return State.TERMINATED_CONNECTIVITY_ERROR;
174 }
175 }
176
177 @Override
178 synchronized void deliverPacket(final JinglePacket jinglePacket) {
179 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
180 switch (jinglePacket.getAction()) {
181 case SESSION_INITIATE:
182 receiveSessionInitiate(jinglePacket);
183 break;
184 case TRANSPORT_INFO:
185 receiveTransportInfo(jinglePacket);
186 break;
187 case SESSION_ACCEPT:
188 receiveSessionAccept(jinglePacket);
189 break;
190 case SESSION_TERMINATE:
191 receiveSessionTerminate(jinglePacket);
192 break;
193 default:
194 respondOk(jinglePacket);
195 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
196 break;
197 }
198 }
199
200 @Override
201 synchronized void notifyRebound() {
202 if (isTerminated()) {
203 return;
204 }
205 webRTCWrapper.close();
206 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
207 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
208 }
209 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
210 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
211 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
212 } else {
213 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
214 finish();
215 }
216 }
217
218 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
219 respondOk(jinglePacket);
220 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
221 final State previous = this.state;
222 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
223 if (TERMINATED.contains(previous)) {
224 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
225 return;
226 }
227 webRTCWrapper.close();
228 final State target = reasonToState(wrapper.reason);
229 transitionOrThrow(target);
230 writeLogMessage(target);
231 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
232 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
233 }
234 finish();
235 }
236
237 private void receiveTransportInfo(final JinglePacket jinglePacket) {
238 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
239 respondOk(jinglePacket);
240 final RtpContentMap contentMap;
241 try {
242 contentMap = RtpContentMap.of(jinglePacket);
243 } catch (IllegalArgumentException | NullPointerException e) {
244 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
245 return;
246 }
247 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
248 if (this.state == State.SESSION_ACCEPTED) {
249 try {
250 processCandidates(candidates);
251 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
252 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
253 }
254 } else {
255 pendingIceCandidates.push(candidates);
256 }
257 } else {
258 if (isTerminated()) {
259 respondOk(jinglePacket);
260 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
261 } else {
262 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
263 terminateWithOutOfOrder(jinglePacket);
264 }
265 }
266 }
267
268 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
269 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
270 final Group originalGroup = rtpContentMap.group;
271 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
272 if (identificationTags.size() == 0) {
273 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
274 }
275 processCandidates(identificationTags, contents);
276 }
277
278 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
279 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
280 final String ufrag = content.getValue().transport.getAttribute("ufrag");
281 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
282 final String sdp;
283 try {
284 sdp = candidate.toSdpAttribute(ufrag);
285 } catch (IllegalArgumentException e) {
286 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
287 continue;
288 }
289 final String sdpMid = content.getKey();
290 final int mLineIndex = indices.indexOf(sdpMid);
291 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
292 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
293 this.webRTCWrapper.addIceCandidate(iceCandidate);
294 }
295 }
296 }
297
298 private RtpContentMap receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
299 final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket);
300 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
301 final AxolotlService.OmemoVerifiedPayload<RtpContentMap> omemoVerifiedPayload;
302 try {
303 omemoVerifiedPayload = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
304 } catch (final CryptoFailedException e) {
305 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
306 }
307 this.omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
308 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received verifiable DTLS fingerprint via "+this.omemoVerification);
309 return omemoVerifiedPayload.getPayload();
310 } else if (expectVerification) {
311 throw new SecurityException("DTLS fingerprint was unexpectedly not verifiable");
312 } else {
313 return receivedContentMap;
314 }
315 }
316
317 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
318 if (isInitiator()) {
319 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
320 terminateWithOutOfOrder(jinglePacket);
321 return;
322 }
323 final RtpContentMap contentMap;
324 try {
325 contentMap = receiveRtpContentMap(jinglePacket, false);
326 contentMap.requireContentDescriptions();
327 contentMap.requireDTLSFingerprint();
328 } catch (final RuntimeException e) {
329 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
330 respondOk(jinglePacket);
331 sendSessionTerminate(Reason.of(e), e.getMessage());
332 return;
333 }
334 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
335 final State target;
336 if (this.state == State.PROCEED) {
337 Preconditions.checkState(
338 proposedMedia != null && proposedMedia.size() > 0,
339 "proposed media must be set when processing pre-approved session-initiate"
340 );
341 if (!this.proposedMedia.equals(contentMap.getMedia())) {
342 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
343 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
344 this.proposedMedia,
345 contentMap.getMedia()
346 ));
347 return;
348 }
349 target = State.SESSION_INITIALIZED_PRE_APPROVED;
350 } else {
351 target = State.SESSION_INITIALIZED;
352 }
353 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
354 respondOk(jinglePacket);
355 //TODO Do not push empty set
356 pendingIceCandidates.push(contentMap.contents.entrySet());
357 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
358 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
359 sendSessionAccept();
360 } else {
361 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
362 startRinging();
363 }
364 } else {
365 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
366 terminateWithOutOfOrder(jinglePacket);
367 }
368 }
369
370 private void receiveSessionAccept(final JinglePacket jinglePacket) {
371 if (!isInitiator()) {
372 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
373 terminateWithOutOfOrder(jinglePacket);
374 return;
375 }
376 final RtpContentMap contentMap;
377 try {
378 contentMap = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
379 contentMap.requireContentDescriptions();
380 contentMap.requireDTLSFingerprint();
381 } catch (final RuntimeException e) {
382 respondOk(jinglePacket);
383 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
384 webRTCWrapper.close();
385 sendSessionTerminate(Reason.of(e), e.getMessage());
386 return;
387 }
388 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
389 if (!initiatorMedia.equals(contentMap.getMedia())) {
390 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
391 "Your session-included included media %s but our session-initiate was %s",
392 this.proposedMedia,
393 contentMap.getMedia()
394 ));
395 return;
396 }
397 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
398 if (transition(State.SESSION_ACCEPTED)) {
399 respondOk(jinglePacket);
400 receiveSessionAccept(contentMap);
401 } else {
402 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
403 respondOk(jinglePacket);
404 }
405 }
406
407 private void receiveSessionAccept(final RtpContentMap contentMap) {
408 this.responderRtpContentMap = contentMap;
409 final SessionDescription sessionDescription;
410 try {
411 sessionDescription = SessionDescription.of(contentMap);
412 } catch (final IllegalArgumentException | NullPointerException e) {
413 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
414 webRTCWrapper.close();
415 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
416 return;
417 }
418 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
419 org.webrtc.SessionDescription.Type.ANSWER,
420 sessionDescription.toString()
421 );
422 try {
423 this.webRTCWrapper.setRemoteDescription(answer).get();
424 } catch (final Exception e) {
425 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
426 webRTCWrapper.close();
427 sendSessionTerminate(Reason.FAILED_APPLICATION);
428 return;
429 }
430 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
431 processCandidates(identificationTags, contentMap.contents.entrySet());
432 }
433
434 private void sendSessionAccept() {
435 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
436 if (rtpContentMap == null) {
437 throw new IllegalStateException("initiator RTP Content Map has not been set");
438 }
439 final SessionDescription offer;
440 try {
441 offer = SessionDescription.of(rtpContentMap);
442 } catch (final IllegalArgumentException | NullPointerException e) {
443 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
444 webRTCWrapper.close();
445 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
446 return;
447 }
448 sendSessionAccept(rtpContentMap.getMedia(), offer);
449 }
450
451 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
452 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
453 }
454
455 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
456 if (isTerminated()) {
457 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
458 return;
459 }
460 try {
461 setupWebRTC(media, iceServers);
462 } catch (final WebRTCWrapper.InitializationException e) {
463 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
464 webRTCWrapper.close();
465 sendSessionTerminate(Reason.FAILED_APPLICATION);
466 return;
467 }
468 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
469 org.webrtc.SessionDescription.Type.OFFER,
470 offer.toString()
471 );
472 try {
473 this.webRTCWrapper.setRemoteDescription(sdp).get();
474 addIceCandidatesFromBlackLog();
475 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
476 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
477 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
478 sendSessionAccept(respondingRtpContentMap);
479 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
480 } catch (final Exception e) {
481 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(e));
482 webRTCWrapper.close();
483 sendSessionTerminate(Reason.FAILED_APPLICATION);
484 }
485 }
486
487 private void addIceCandidatesFromBlackLog() {
488 while (!this.pendingIceCandidates.isEmpty()) {
489 processCandidates(this.pendingIceCandidates.poll());
490 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
491 }
492 }
493
494 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
495 this.responderRtpContentMap = rtpContentMap;
496 this.transitionOrThrow(State.SESSION_ACCEPTED);
497 final RtpContentMap outgoingContentMap;
498 if (this.omemoVerification.hasDeviceId()) {
499 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
500 try {
501 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
502 outgoingContentMap = verifiedPayload.getPayload();
503 this.omemoVerification.setOrEnsureEqual(verifiedPayload);
504 } catch (final Exception e) {
505 throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e);
506 }
507 } else {
508 outgoingContentMap = rtpContentMap;
509 }
510 final JinglePacket sessionAccept = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
511 send(sessionAccept);
512 }
513
514 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
515 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
516 switch (message.getName()) {
517 case "propose":
518 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
519 break;
520 case "proceed":
521 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
522 break;
523 case "retract":
524 receiveRetract(from, serverMessageId, timestamp);
525 break;
526 case "reject":
527 receiveReject(from, serverMessageId, timestamp);
528 break;
529 case "accept":
530 receiveAccept(from, serverMessageId, timestamp);
531 break;
532 default:
533 break;
534 }
535 }
536
537 void deliverFailedProceed() {
538 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
539 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
540 webRTCWrapper.close();
541 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
542 this.finish();
543 }
544 }
545
546 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
547 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
548 if (originatedFromMyself) {
549 if (transition(State.ACCEPTED)) {
550 if (serverMsgId != null) {
551 this.message.setServerMsgId(serverMsgId);
552 }
553 this.message.setTime(timestamp);
554 this.message.setCarbon(true); //indicate that call was accepted on other device
555 this.writeLogMessageSuccess(0);
556 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
557 this.finish();
558 } else {
559 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
560 }
561 } else {
562 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
563 }
564 }
565
566 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
567 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
568 //reject from another one of my clients
569 if (originatedFromMyself) {
570 receiveRejectFromMyself(serverMsgId, timestamp);
571 } else if (isInitiator()) {
572 if (from.equals(id.with)) {
573 receiveRejectFromResponder();
574 } else {
575 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
576 }
577 } else {
578 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
579 }
580 }
581
582 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
583 if (transition(State.REJECTED)) {
584 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
585 this.finish();
586 if (serverMsgId != null) {
587 this.message.setServerMsgId(serverMsgId);
588 }
589 this.message.setTime(timestamp);
590 this.message.setCarbon(true); //indicate that call was rejected on other device
591 writeLogMessageMissed();
592 } else {
593 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
594 }
595 }
596
597 private void receiveRejectFromResponder() {
598 if (isInState(State.PROCEED)) {
599 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
600 closeTransitionLogFinish(State.REJECTED_RACED);
601 return;
602 }
603 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
604 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
605 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
606 return;
607 }
608 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
609 }
610
611 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
612 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
613 if (originatedFromMyself) {
614 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
615 } else if (transition(State.PROPOSED, () -> {
616 final Collection<RtpDescription> descriptions = Collections2.transform(
617 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
618 input -> (RtpDescription) input
619 );
620 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
621 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
622 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
623 this.proposedMedia = Sets.newHashSet(media);
624 })) {
625 if (serverMsgId != null) {
626 this.message.setServerMsgId(serverMsgId);
627 }
628 this.message.setTime(timestamp);
629 startRinging();
630 } else {
631 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
632 }
633 }
634
635 private void startRinging() {
636 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
637 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
638 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
639 }
640
641 private synchronized void ringingTimeout() {
642 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
643 switch (this.state) {
644 case PROPOSED:
645 message.markUnread();
646 rejectCallFromProposed();
647 break;
648 case SESSION_INITIALIZED:
649 message.markUnread();
650 rejectCallFromSessionInitiate();
651 break;
652 }
653 }
654
655 private void cancelRingingTimeout() {
656 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
657 if (future != null && !future.isCancelled()) {
658 future.cancel(false);
659 }
660 }
661
662 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
663 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
664 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
665 if (from.equals(id.with)) {
666 if (isInitiator()) {
667 if (transition(State.PROCEED)) {
668 if (serverMsgId != null) {
669 this.message.setServerMsgId(serverMsgId);
670 }
671 this.message.setTime(timestamp);
672 this.omemoVerification.setDeviceId(proceed.getDeviceId());
673 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
674 } else {
675 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
676 }
677 } else {
678 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
679 }
680 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
681 if (transition(State.ACCEPTED)) {
682 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
683 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
684 this.finish();
685 }
686 } else {
687 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
688 }
689 }
690
691 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
692 if (from.equals(id.with)) {
693 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
694 if (transition(target)) {
695 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
696 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
697 if (serverMsgId != null) {
698 this.message.setServerMsgId(serverMsgId);
699 }
700 this.message.setTime(timestamp);
701 if (target == State.RETRACTED) {
702 this.message.markUnread();
703 }
704 writeLogMessageMissed();
705 finish();
706 } else {
707 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
708 }
709 } else {
710 //TODO parse retract from self
711 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
712 }
713 }
714
715 public void sendSessionInitiate() {
716 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
717 }
718
719 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
720 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
721 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
722 }
723
724 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
725 if (isTerminated()) {
726 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
727 return;
728 }
729 try {
730 setupWebRTC(media, iceServers);
731 } catch (final WebRTCWrapper.InitializationException e) {
732 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
733 webRTCWrapper.close();
734 sendJingleMessage("retract", id.with.asBareJid());
735 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
736 this.finish();
737 return;
738 }
739 try {
740 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
741 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
742 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
743 sendSessionInitiate(rtpContentMap, targetState);
744 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
745 } catch (final Exception e) {
746 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(e));
747 webRTCWrapper.close();
748 if (isInState(targetState)) {
749 sendSessionTerminate(Reason.FAILED_APPLICATION);
750 } else {
751 sendJingleMessage("retract", id.with.asBareJid());
752 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
753 this.finish();
754 }
755 }
756 }
757
758 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
759 this.initiatorRtpContentMap = rtpContentMap;
760 this.transitionOrThrow(targetState);
761 //TODO do on background thread?
762 final RtpContentMap outgoingContentMap = encryptSessionInitiate(rtpContentMap);
763 final JinglePacket sessionInitiate = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
764 send(sessionInitiate);
765 }
766
767 private RtpContentMap encryptSessionInitiate(final RtpContentMap rtpContentMap) {
768 if (this.omemoVerification.hasDeviceId()) {
769 final AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap> verifiedPayload;
770 try {
771 verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
772 } catch (final CryptoFailedException e) {
773 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
774 return rtpContentMap;
775 }
776 this.omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
777 return verifiedPayload.getPayload();
778 } else {
779 return rtpContentMap;
780 }
781 }
782
783 private void sendSessionTerminate(final Reason reason) {
784 sendSessionTerminate(reason, null);
785 }
786
787 private void sendSessionTerminate(final Reason reason, final String text) {
788 final State previous = this.state;
789 final State target = reasonToState(reason);
790 transitionOrThrow(target);
791 if (previous != State.NULL) {
792 writeLogMessage(target);
793 }
794 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
795 jinglePacket.setReason(reason, text);
796 Log.d(Config.LOGTAG, jinglePacket.toString());
797 send(jinglePacket);
798 finish();
799 }
800
801 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
802 final RtpContentMap transportInfo;
803 try {
804 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
805 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
806 } catch (final Exception e) {
807 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
808 return;
809 }
810 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
811 send(jinglePacket);
812 }
813
814 private void send(final JinglePacket jinglePacket) {
815 jinglePacket.setTo(id.with);
816 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
817 }
818
819 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
820 if (response.getType() == IqPacket.TYPE.ERROR) {
821 final String errorCondition = response.getErrorCondition();
822 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
823 if (isTerminated()) {
824 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
825 return;
826 }
827 this.webRTCWrapper.close();
828 final State target;
829 if (Arrays.asList(
830 "service-unavailable",
831 "recipient-unavailable",
832 "remote-server-not-found",
833 "remote-server-timeout"
834 ).contains(errorCondition)) {
835 target = State.TERMINATED_CONNECTIVITY_ERROR;
836 } else {
837 target = State.TERMINATED_APPLICATION_FAILURE;
838 }
839 transitionOrThrow(target);
840 this.finish();
841 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
842 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
843 if (isTerminated()) {
844 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
845 return;
846 }
847 this.webRTCWrapper.close();
848 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
849 this.finish();
850 }
851 }
852
853 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
854 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
855 this.webRTCWrapper.close();
856 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
857 respondWithOutOfOrder(jinglePacket);
858 this.finish();
859 }
860
861 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
862 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
863 }
864
865 private void respondOk(final JinglePacket jinglePacket) {
866 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
867 }
868
869 public void throwStateTransitionException() {
870 final StateTransitionException exception = this.stateTransitionException;
871 if (exception != null) {
872 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
873 }
874 }
875
876 public RtpEndUserState getEndUserState() {
877 switch (this.state) {
878 case NULL:
879 case PROPOSED:
880 case SESSION_INITIALIZED:
881 if (isInitiator()) {
882 return RtpEndUserState.RINGING;
883 } else {
884 return RtpEndUserState.INCOMING_CALL;
885 }
886 case PROCEED:
887 if (isInitiator()) {
888 return RtpEndUserState.RINGING;
889 } else {
890 return RtpEndUserState.ACCEPTING_CALL;
891 }
892 case SESSION_INITIALIZED_PRE_APPROVED:
893 if (isInitiator()) {
894 return RtpEndUserState.RINGING;
895 } else {
896 return RtpEndUserState.CONNECTING;
897 }
898 case SESSION_ACCEPTED:
899 final PeerConnection.PeerConnectionState state;
900 try {
901 state = webRTCWrapper.getState();
902 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
903 //We usually close the WebRTCWrapper *before* transitioning so we might still
904 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
905 return RtpEndUserState.ENDING_CALL;
906 }
907 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
908 return RtpEndUserState.CONNECTED;
909 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
910 return RtpEndUserState.CONNECTING;
911 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
912 return RtpEndUserState.ENDING_CALL;
913 } else {
914 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
915 }
916 case REJECTED:
917 case REJECTED_RACED:
918 case TERMINATED_DECLINED_OR_BUSY:
919 if (isInitiator()) {
920 return RtpEndUserState.DECLINED_OR_BUSY;
921 } else {
922 return RtpEndUserState.ENDED;
923 }
924 case TERMINATED_SUCCESS:
925 case ACCEPTED:
926 case RETRACTED:
927 case TERMINATED_CANCEL_OR_TIMEOUT:
928 return RtpEndUserState.ENDED;
929 case RETRACTED_RACED:
930 if (isInitiator()) {
931 return RtpEndUserState.ENDED;
932 } else {
933 return RtpEndUserState.RETRACTED;
934 }
935 case TERMINATED_CONNECTIVITY_ERROR:
936 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
937 case TERMINATED_APPLICATION_FAILURE:
938 return RtpEndUserState.APPLICATION_ERROR;
939 }
940 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
941 }
942
943 public Set<Media> getMedia() {
944 final State current = getState();
945 if (current == State.NULL) {
946 if (isInitiator()) {
947 return Preconditions.checkNotNull(
948 this.proposedMedia,
949 "RTP connection has not been initialized properly"
950 );
951 }
952 throw new IllegalStateException("RTP connection has not been initialized yet");
953 }
954 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
955 return Preconditions.checkNotNull(
956 this.proposedMedia,
957 "RTP connection has not been initialized properly"
958 );
959 }
960 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
961 if (initiatorContentMap != null) {
962 return initiatorContentMap.getMedia();
963 } else if (isTerminated()) {
964 return Collections.emptySet(); //we might fail before we ever got a chance to set media
965 } else {
966 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
967 }
968 }
969
970
971 public boolean isVerified() {
972 final String fingerprint = this.omemoVerification.getFingerprint();
973 if (fingerprint == null) {
974 return false;
975 }
976 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
977 return status != null && status.getTrust() == FingerprintStatus.Trust.VERIFIED;
978 }
979
980 public synchronized void acceptCall() {
981 switch (this.state) {
982 case PROPOSED:
983 cancelRingingTimeout();
984 acceptCallFromProposed();
985 break;
986 case SESSION_INITIALIZED:
987 cancelRingingTimeout();
988 acceptCallFromSessionInitialized();
989 break;
990 case ACCEPTED:
991 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
992 break;
993 case PROCEED:
994 case SESSION_ACCEPTED:
995 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
996 break;
997 default:
998 throw new IllegalStateException("Can not accept call from " + this.state);
999 }
1000 }
1001
1002
1003 public void notifyPhoneCall() {
1004 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1005 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1006 rejectCall();
1007 } else {
1008 endCall();
1009 }
1010 }
1011
1012 public synchronized void rejectCall() {
1013 if (isTerminated()) {
1014 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1015 return;
1016 }
1017 switch (this.state) {
1018 case PROPOSED:
1019 rejectCallFromProposed();
1020 break;
1021 case SESSION_INITIALIZED:
1022 rejectCallFromSessionInitiate();
1023 break;
1024 default:
1025 throw new IllegalStateException("Can not reject call from " + this.state);
1026 }
1027 }
1028
1029 public synchronized void endCall() {
1030 if (isTerminated()) {
1031 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1032 return;
1033 }
1034 if (isInState(State.PROPOSED) && !isInitiator()) {
1035 rejectCallFromProposed();
1036 return;
1037 }
1038 if (isInState(State.PROCEED)) {
1039 if (isInitiator()) {
1040 retractFromProceed();
1041 } else {
1042 rejectCallFromProceed();
1043 }
1044 return;
1045 }
1046 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1047 this.webRTCWrapper.close();
1048 sendSessionTerminate(Reason.CANCEL);
1049 return;
1050 }
1051 if (isInState(State.SESSION_INITIALIZED)) {
1052 rejectCallFromSessionInitiate();
1053 return;
1054 }
1055 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1056 this.webRTCWrapper.close();
1057 sendSessionTerminate(Reason.SUCCESS);
1058 return;
1059 }
1060 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1061 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1062 return;
1063 }
1064 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1065 }
1066
1067 private void retractFromProceed() {
1068 Log.d(Config.LOGTAG, "retract from proceed");
1069 this.sendJingleMessage("retract");
1070 closeTransitionLogFinish(State.RETRACTED_RACED);
1071 }
1072
1073 private void closeTransitionLogFinish(final State state) {
1074 this.webRTCWrapper.close();
1075 transitionOrThrow(state);
1076 writeLogMessage(state);
1077 finish();
1078 }
1079
1080 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1081 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1082 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1083 if (media.contains(Media.VIDEO)) {
1084 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1085 } else {
1086 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1087 }
1088 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1089 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1090 }
1091
1092 private void acceptCallFromProposed() {
1093 transitionOrThrow(State.PROCEED);
1094 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1095 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1096 this.sendJingleMessage("proceed");
1097 }
1098
1099 private void rejectCallFromProposed() {
1100 transitionOrThrow(State.REJECTED);
1101 writeLogMessageMissed();
1102 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1103 this.sendJingleMessage("reject");
1104 finish();
1105 }
1106
1107 private void rejectCallFromProceed() {
1108 this.sendJingleMessage("reject");
1109 closeTransitionLogFinish(State.REJECTED_RACED);
1110 }
1111
1112 private void rejectCallFromSessionInitiate() {
1113 webRTCWrapper.close();
1114 sendSessionTerminate(Reason.DECLINE);
1115 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1116 }
1117
1118 private void sendJingleMessage(final String action) {
1119 sendJingleMessage(action, id.with);
1120 }
1121
1122 private void sendJingleMessage(final String action, final Jid to) {
1123 final MessagePacket messagePacket = new MessagePacket();
1124 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1125 messagePacket.setTo(to);
1126 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1127 if ("proceed".equals(action)) {
1128 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1129
1130 //TODO only do this if OMEMO is enable so we have an easy way to opt out
1131 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1132 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1133 device.setAttribute("id", deviceId);
1134 }
1135 messagePacket.addChild("store", "urn:xmpp:hints");
1136 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1137 }
1138
1139 private void acceptCallFromSessionInitialized() {
1140 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1141 sendSessionAccept();
1142 }
1143
1144 private synchronized boolean isInState(State... state) {
1145 return Arrays.asList(state).contains(this.state);
1146 }
1147
1148 private boolean transition(final State target) {
1149 return transition(target, null);
1150 }
1151
1152 private synchronized boolean transition(final State target, final Runnable runnable) {
1153 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1154 if (validTransitions != null && validTransitions.contains(target)) {
1155 this.state = target;
1156 this.stateTransitionException = new StateTransitionException(target);
1157 if (runnable != null) {
1158 runnable.run();
1159 }
1160 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1161 updateEndUserState();
1162 updateOngoingCallNotification();
1163 return true;
1164 } else {
1165 return false;
1166 }
1167 }
1168
1169 void transitionOrThrow(final State target) {
1170 if (!transition(target)) {
1171 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1172 }
1173 }
1174
1175 @Override
1176 public void onIceCandidate(final IceCandidate iceCandidate) {
1177 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1178 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1179 sendTransportInfo(iceCandidate.sdpMid, candidate);
1180 }
1181
1182 @Override
1183 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1184 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1185 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
1186 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
1187 }
1188 if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) {
1189 this.rtpConnectionEnded = SystemClock.elapsedRealtime();
1190 }
1191 //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace
1192 //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable
1193 //as there is no content-replace
1194 if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) {
1195 if (isTerminated()) {
1196 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1197 return;
1198 }
1199 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1200 } else {
1201 updateEndUserState();
1202 }
1203 }
1204
1205 private void closeWebRTCSessionAfterFailedConnection() {
1206 this.webRTCWrapper.close();
1207 synchronized (this) {
1208 if (isTerminated()) {
1209 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1210 return;
1211 }
1212 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1213 }
1214 }
1215
1216 public long getRtpConnectionStarted() {
1217 return this.rtpConnectionStarted;
1218 }
1219
1220 public long getRtpConnectionEnded() {
1221 return this.rtpConnectionEnded;
1222 }
1223
1224 public AppRTCAudioManager getAudioManager() {
1225 return webRTCWrapper.getAudioManager();
1226 }
1227
1228 public boolean isMicrophoneEnabled() {
1229 return webRTCWrapper.isMicrophoneEnabled();
1230 }
1231
1232 public boolean setMicrophoneEnabled(final boolean enabled) {
1233 return webRTCWrapper.setMicrophoneEnabled(enabled);
1234 }
1235
1236 public boolean isVideoEnabled() {
1237 return webRTCWrapper.isVideoEnabled();
1238 }
1239
1240 public void setVideoEnabled(final boolean enabled) {
1241 webRTCWrapper.setVideoEnabled(enabled);
1242 }
1243
1244 public boolean isCameraSwitchable() {
1245 return webRTCWrapper.isCameraSwitchable();
1246 }
1247
1248 public boolean isFrontCamera() {
1249 return webRTCWrapper.isFrontCamera();
1250 }
1251
1252 public ListenableFuture<Boolean> switchCamera() {
1253 return webRTCWrapper.switchCamera();
1254 }
1255
1256 @Override
1257 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1258 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1259 }
1260
1261 private void updateEndUserState() {
1262 final RtpEndUserState endUserState = getEndUserState();
1263 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1264 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1265 }
1266
1267 private void updateOngoingCallNotification() {
1268 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1269 xmppConnectionService.setOngoingCall(id, getMedia());
1270 } else {
1271 xmppConnectionService.removeOngoingCall();
1272 }
1273 }
1274
1275 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1276 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1277 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1278 request.setTo(id.account.getDomain());
1279 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1280 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1281 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1282 if (response.getType() == IqPacket.TYPE.RESULT) {
1283 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1284 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1285 for (final Element child : children) {
1286 if ("service".equals(child.getName())) {
1287 final String type = child.getAttribute("type");
1288 final String host = child.getAttribute("host");
1289 final String sport = child.getAttribute("port");
1290 final Integer port = sport == null ? null : Ints.tryParse(sport);
1291 final String transport = child.getAttribute("transport");
1292 final String username = child.getAttribute("username");
1293 final String password = child.getAttribute("password");
1294 if (Strings.isNullOrEmpty(host) || port == null) {
1295 continue;
1296 }
1297 if (port < 0 || port > 65535) {
1298 continue;
1299 }
1300 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1301 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1302 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1303 continue;
1304 }
1305 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1306 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1307 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1308 if (username != null && password != null) {
1309 iceServerBuilder.setUsername(username);
1310 iceServerBuilder.setPassword(password);
1311 } else if (Arrays.asList("turn", "turns").contains(type)) {
1312 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1313 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1314 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1315 continue;
1316 }
1317 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1318 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1319 listBuilder.add(iceServer);
1320 }
1321 }
1322 }
1323 }
1324 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1325 if (iceServers.size() == 0) {
1326 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1327 }
1328 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1329 });
1330 } else {
1331 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1332 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1333 }
1334 }
1335
1336 private void finish() {
1337 if (isTerminated()) {
1338 this.cancelRingingTimeout();
1339 this.webRTCWrapper.verifyClosed();
1340 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1341 this.jingleConnectionManager.finishConnectionOrThrow(this);
1342 } else {
1343 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1344 }
1345 }
1346
1347 private void writeLogMessage(final State state) {
1348 final long started = this.rtpConnectionStarted;
1349 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
1350 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1351 writeLogMessageSuccess(duration);
1352 } else {
1353 writeLogMessageMissed();
1354 }
1355 }
1356
1357 private void writeLogMessageSuccess(final long duration) {
1358 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1359 this.writeMessage();
1360 }
1361
1362 private void writeLogMessageMissed() {
1363 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1364 this.writeMessage();
1365 }
1366
1367 private void writeMessage() {
1368 final Conversational conversational = message.getConversation();
1369 if (conversational instanceof Conversation) {
1370 ((Conversation) conversational).add(this.message);
1371 xmppConnectionService.createMessageAsync(message);
1372 xmppConnectionService.updateConversationUi();
1373 } else {
1374 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1375 }
1376 }
1377
1378 public State getState() {
1379 return this.state;
1380 }
1381
1382 boolean isTerminated() {
1383 return TERMINATED.contains(this.state);
1384 }
1385
1386 public Optional<VideoTrack> getLocalVideoTrack() {
1387 return webRTCWrapper.getLocalVideoTrack();
1388 }
1389
1390 public Optional<VideoTrack> getRemoteVideoTrack() {
1391 return webRTCWrapper.getRemoteVideoTrack();
1392 }
1393
1394
1395 public EglBase.Context getEglBaseContext() {
1396 return webRTCWrapper.getEglBaseContext();
1397 }
1398
1399 void setProposedMedia(final Set<Media> media) {
1400 this.proposedMedia = media;
1401 }
1402
1403 public void fireStateUpdate() {
1404 final RtpEndUserState endUserState = getEndUserState();
1405 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1406 }
1407
1408 private interface OnIceServersDiscovered {
1409 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1410 }
1411
1412 private static class StateTransitionException extends Exception {
1413 private final State state;
1414
1415 private StateTransitionException(final State state) {
1416 this.state = state;
1417 }
1418 }
1419}