1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.EglBase;
24import org.webrtc.IceCandidate;
25import org.webrtc.PeerConnection;
26import org.webrtc.VideoTrack;
27
28import java.util.ArrayDeque;
29import java.util.Arrays;
30import java.util.Collection;
31import java.util.Collections;
32import java.util.LinkedList;
33import java.util.List;
34import java.util.Map;
35import java.util.Queue;
36import java.util.Set;
37import java.util.concurrent.ExecutionException;
38import java.util.concurrent.ScheduledFuture;
39import java.util.concurrent.TimeUnit;
40
41import eu.siacs.conversations.Config;
42import eu.siacs.conversations.crypto.axolotl.AxolotlService;
43import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
44import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
45import eu.siacs.conversations.entities.Account;
46import eu.siacs.conversations.entities.Conversation;
47import eu.siacs.conversations.entities.Conversational;
48import eu.siacs.conversations.entities.Message;
49import eu.siacs.conversations.entities.RtpSessionStatus;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51import eu.siacs.conversations.utils.IP;
52import eu.siacs.conversations.xml.Element;
53import eu.siacs.conversations.xml.Namespace;
54import eu.siacs.conversations.xmpp.Jid;
55import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
56import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
57import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
61import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
62import eu.siacs.conversations.xmpp.stanzas.IqPacket;
63import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
64
65public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
66
67 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
68 State.PROCEED,
69 State.SESSION_INITIALIZED_PRE_APPROVED,
70 State.SESSION_ACCEPTED
71 );
72 private static final long BUSY_TIME_OUT = 30;
73 private static final List<State> TERMINATED = Arrays.asList(
74 State.ACCEPTED,
75 State.REJECTED,
76 State.REJECTED_RACED,
77 State.RETRACTED,
78 State.RETRACTED_RACED,
79 State.TERMINATED_SUCCESS,
80 State.TERMINATED_DECLINED_OR_BUSY,
81 State.TERMINATED_CONNECTIVITY_ERROR,
82 State.TERMINATED_CANCEL_OR_TIMEOUT,
83 State.TERMINATED_APPLICATION_FAILURE,
84 State.TERMINATED_SECURITY_ERROR
85 );
86
87 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
88
89 static {
90 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
91 transitionBuilder.put(State.NULL, ImmutableList.of(
92 State.PROPOSED,
93 State.SESSION_INITIALIZED,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR
96 ));
97 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
98 State.ACCEPTED,
99 State.PROCEED,
100 State.REJECTED,
101 State.RETRACTED,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
105 ));
106 transitionBuilder.put(State.PROCEED, ImmutableList.of(
107 State.REJECTED_RACED,
108 State.RETRACTED_RACED,
109 State.SESSION_INITIALIZED_PRE_APPROVED,
110 State.TERMINATED_SUCCESS,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR,
113 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
125 State.SESSION_ACCEPTED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_DECLINED_OR_BUSY,
128 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
129 State.TERMINATED_CANCEL_OR_TIMEOUT,
130 State.TERMINATED_APPLICATION_FAILURE,
131 State.TERMINATED_SECURITY_ERROR
132 ));
133 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
134 State.TERMINATED_SUCCESS,
135 State.TERMINATED_DECLINED_OR_BUSY,
136 State.TERMINATED_CONNECTIVITY_ERROR,
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR
140 ));
141 VALID_TRANSITIONS = transitionBuilder.build();
142 }
143
144 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
145 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
146 private final OmemoVerification omemoVerification = new OmemoVerification();
147 private final Message message;
148 private State state = State.NULL;
149 private StateTransitionException stateTransitionException;
150 private Set<Media> proposedMedia;
151 private RtpContentMap initiatorRtpContentMap;
152 private RtpContentMap responderRtpContentMap;
153 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
154 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
155 private ScheduledFuture<?> ringingTimeoutFuture;
156
157 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
158 super(jingleConnectionManager, id, initiator);
159 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
160 id.account,
161 id.with.asBareJid(),
162 false,
163 false
164 );
165 this.message = new Message(
166 conversation,
167 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
168 Message.TYPE_RTP_SESSION,
169 id.sessionId
170 );
171 }
172
173 private static State reasonToState(Reason reason) {
174 switch (reason) {
175 case SUCCESS:
176 return State.TERMINATED_SUCCESS;
177 case DECLINE:
178 case BUSY:
179 return State.TERMINATED_DECLINED_OR_BUSY;
180 case CANCEL:
181 case TIMEOUT:
182 return State.TERMINATED_CANCEL_OR_TIMEOUT;
183 case SECURITY_ERROR:
184 return State.TERMINATED_SECURITY_ERROR;
185 case FAILED_APPLICATION:
186 case UNSUPPORTED_TRANSPORTS:
187 case UNSUPPORTED_APPLICATIONS:
188 return State.TERMINATED_APPLICATION_FAILURE;
189 default:
190 return State.TERMINATED_CONNECTIVITY_ERROR;
191 }
192 }
193
194 @Override
195 synchronized void deliverPacket(final JinglePacket jinglePacket) {
196 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
197 switch (jinglePacket.getAction()) {
198 case SESSION_INITIATE:
199 receiveSessionInitiate(jinglePacket);
200 break;
201 case TRANSPORT_INFO:
202 receiveTransportInfo(jinglePacket);
203 break;
204 case SESSION_ACCEPT:
205 receiveSessionAccept(jinglePacket);
206 break;
207 case SESSION_TERMINATE:
208 receiveSessionTerminate(jinglePacket);
209 break;
210 default:
211 respondOk(jinglePacket);
212 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
213 break;
214 }
215 }
216
217 @Override
218 synchronized void notifyRebound() {
219 if (isTerminated()) {
220 return;
221 }
222 webRTCWrapper.close();
223 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
227 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
228 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
229 } else {
230 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
231 finish();
232 }
233 }
234
235 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
236 respondOk(jinglePacket);
237 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
238 final State previous = this.state;
239 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
240 if (TERMINATED.contains(previous)) {
241 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
242 return;
243 }
244 webRTCWrapper.close();
245 final State target = reasonToState(wrapper.reason);
246 transitionOrThrow(target);
247 writeLogMessage(target);
248 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
249 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
250 }
251 finish();
252 }
253
254 private void receiveTransportInfo(final JinglePacket jinglePacket) {
255 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
256 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
257 respondOk(jinglePacket);
258 final RtpContentMap contentMap;
259 try {
260 contentMap = RtpContentMap.of(jinglePacket);
261 } catch (IllegalArgumentException | NullPointerException e) {
262 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
263 return;
264 }
265 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
266 if (this.state == State.SESSION_ACCEPTED) {
267 try {
268 processCandidates(candidates);
269 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
270 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
271 }
272 } else {
273 pendingIceCandidates.push(candidates);
274 }
275 } else {
276 if (isTerminated()) {
277 respondOk(jinglePacket);
278 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
279 } else {
280 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
281 terminateWithOutOfOrder(jinglePacket);
282 }
283 }
284 }
285
286 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
287 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
288 final Group originalGroup = rtpContentMap.group;
289 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
290 if (identificationTags.size() == 0) {
291 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
292 }
293 processCandidates(identificationTags, contents);
294 }
295
296 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
297 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
298 final String ufrag = content.getValue().transport.getAttribute("ufrag");
299 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
300 final String sdp;
301 try {
302 sdp = candidate.toSdpAttribute(ufrag);
303 } catch (IllegalArgumentException e) {
304 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
305 continue;
306 }
307 final String sdpMid = content.getKey();
308 final int mLineIndex = indices.indexOf(sdpMid);
309 if (mLineIndex < 0) {
310 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
311 }
312 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
313 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
314 this.webRTCWrapper.addIceCandidate(iceCandidate);
315 }
316 }
317 }
318
319 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
320 final RtpContentMap receivedContentMap;
321 try {
322 receivedContentMap = RtpContentMap.of(jinglePacket);
323 } catch (final Exception e) {
324 return Futures.immediateFailedFuture(e);
325 }
326 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
327 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
328 return Futures.transform(future, omemoVerifiedPayload -> {
329 //TODO test if an exception here triggers a correct abort
330 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
331 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
332 return omemoVerifiedPayload.getPayload();
333 }, MoreExecutors.directExecutor());
334 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
335 return Futures.immediateFailedFuture(
336 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
337 );
338 } else {
339 return Futures.immediateFuture(receivedContentMap);
340 }
341 }
342
343 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
344 if (isInitiator()) {
345 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
346 if (isTerminated()) {
347 Log.d(Config.LOGTAG, String.format(
348 "%s: got a reason to terminate with out-of-order. but already in state %s",
349 id.account.getJid().asBareJid(),
350 getState()
351 ));
352 respondWithOutOfOrder(jinglePacket);
353 } else {
354 terminateWithOutOfOrder(jinglePacket);
355 }
356 return;
357 }
358 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
359 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
360 @Override
361 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
362 receiveSessionInitiate(jinglePacket, rtpContentMap);
363 }
364
365 @Override
366 public void onFailure(@NonNull final Throwable throwable) {
367 respondOk(jinglePacket);
368 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
369 }
370 }, MoreExecutors.directExecutor());
371 }
372
373 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
374 try {
375 contentMap.requireContentDescriptions();
376 contentMap.requireDTLSFingerprint();
377 } catch (final RuntimeException e) {
378 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
379 respondOk(jinglePacket);
380 sendSessionTerminate(Reason.of(e), e.getMessage());
381 return;
382 }
383 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
384 final State target;
385 if (this.state == State.PROCEED) {
386 Preconditions.checkState(
387 proposedMedia != null && proposedMedia.size() > 0,
388 "proposed media must be set when processing pre-approved session-initiate"
389 );
390 if (!this.proposedMedia.equals(contentMap.getMedia())) {
391 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
392 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
393 this.proposedMedia,
394 contentMap.getMedia()
395 ));
396 return;
397 }
398 target = State.SESSION_INITIALIZED_PRE_APPROVED;
399 } else {
400 target = State.SESSION_INITIALIZED;
401 }
402 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
403 respondOk(jinglePacket);
404
405 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
406 if (candidates.size() > 0) {
407 pendingIceCandidates.push(candidates);
408 }
409 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
410 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
411 sendSessionAccept();
412 } else {
413 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
414 startRinging();
415 }
416 } else {
417 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
418 terminateWithOutOfOrder(jinglePacket);
419 }
420 }
421
422 private void receiveSessionAccept(final JinglePacket jinglePacket) {
423 if (!isInitiator()) {
424 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
425 terminateWithOutOfOrder(jinglePacket);
426 return;
427 }
428 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
429 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
430 @Override
431 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
432 receiveSessionAccept(jinglePacket, rtpContentMap);
433 }
434
435 @Override
436 public void onFailure(@NonNull final Throwable throwable) {
437 respondOk(jinglePacket);
438 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
439 webRTCWrapper.close();
440 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
441 }
442 }, MoreExecutors.directExecutor());
443 }
444
445 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
446 try {
447 contentMap.requireContentDescriptions();
448 contentMap.requireDTLSFingerprint();
449 } catch (final RuntimeException e) {
450 respondOk(jinglePacket);
451 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
452 webRTCWrapper.close();
453 sendSessionTerminate(Reason.of(e), e.getMessage());
454 return;
455 }
456 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
457 if (!initiatorMedia.equals(contentMap.getMedia())) {
458 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
459 "Your session-included included media %s but our session-initiate was %s",
460 this.proposedMedia,
461 contentMap.getMedia()
462 ));
463 return;
464 }
465 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
466 if (transition(State.SESSION_ACCEPTED)) {
467 respondOk(jinglePacket);
468 receiveSessionAccept(contentMap);
469 } else {
470 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
471 respondOk(jinglePacket);
472 }
473 }
474
475 private void receiveSessionAccept(final RtpContentMap contentMap) {
476 this.responderRtpContentMap = contentMap;
477 final SessionDescription sessionDescription;
478 try {
479 sessionDescription = SessionDescription.of(contentMap);
480 } catch (final IllegalArgumentException | NullPointerException e) {
481 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
482 webRTCWrapper.close();
483 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
484 return;
485 }
486 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
487 org.webrtc.SessionDescription.Type.ANSWER,
488 sessionDescription.toString()
489 );
490 try {
491 this.webRTCWrapper.setRemoteDescription(answer).get();
492 } catch (final Exception e) {
493 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
494 webRTCWrapper.close();
495 sendSessionTerminate(Reason.FAILED_APPLICATION);
496 return;
497 }
498 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
499 processCandidates(identificationTags, contentMap.contents.entrySet());
500 }
501
502 private void sendSessionAccept() {
503 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
504 if (rtpContentMap == null) {
505 throw new IllegalStateException("initiator RTP Content Map has not been set");
506 }
507 final SessionDescription offer;
508 try {
509 offer = SessionDescription.of(rtpContentMap);
510 } catch (final IllegalArgumentException | NullPointerException e) {
511 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
512 webRTCWrapper.close();
513 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
514 return;
515 }
516 sendSessionAccept(rtpContentMap.getMedia(), offer);
517 }
518
519 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
520 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
521 }
522
523 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
524 if (isTerminated()) {
525 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
526 return;
527 }
528 try {
529 setupWebRTC(media, iceServers);
530 } catch (final WebRTCWrapper.InitializationException e) {
531 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
532 webRTCWrapper.close();
533 sendSessionTerminate(Reason.FAILED_APPLICATION);
534 return;
535 }
536 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
537 org.webrtc.SessionDescription.Type.OFFER,
538 offer.toString()
539 );
540 try {
541 this.webRTCWrapper.setRemoteDescription(sdp).get();
542 addIceCandidatesFromBlackLog();
543 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
544 prepareSessionAccept(webRTCSessionDescription);
545 } catch (final Exception e) {
546 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
547 failureToAcceptSession(e);
548 }
549 }
550
551 private void failureToAcceptSession(final Throwable throwable) {
552 if (isTerminated()) {
553 return;
554 }
555 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(throwable));
556 webRTCWrapper.close();
557 sendSessionTerminate(Reason.ofThrowable(throwable));
558 }
559
560 private void addIceCandidatesFromBlackLog() {
561 while (!this.pendingIceCandidates.isEmpty()) {
562 processCandidates(this.pendingIceCandidates.poll());
563 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
564 }
565 }
566
567 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
568 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
569 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
570 this.responderRtpContentMap = respondingRtpContentMap;
571 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
572 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
573 Futures.addCallback(outgoingContentMapFuture,
574 new FutureCallback<RtpContentMap>() {
575 @Override
576 public void onSuccess(final RtpContentMap outgoingContentMap) {
577 sendSessionAccept(outgoingContentMap);
578 }
579
580 @Override
581 public void onFailure(@NonNull Throwable throwable) {
582 failureToAcceptSession(throwable);
583 }
584 },
585 MoreExecutors.directExecutor()
586 );
587 }
588
589 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
590 if (isTerminated()) {
591 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
592 return;
593 }
594 transitionOrThrow(State.SESSION_ACCEPTED);
595 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
596 send(sessionAccept);
597 }
598
599 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
600 if (this.omemoVerification.hasDeviceId()) {
601 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
602 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
603 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
604 omemoVerification.setOrEnsureEqual(verifiedPayload);
605 return verifiedPayload.getPayload();
606 }, MoreExecutors.directExecutor());
607 } else {
608 return Futures.immediateFuture(rtpContentMap);
609 }
610 }
611
612 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
613 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
614 switch (message.getName()) {
615 case "propose":
616 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
617 break;
618 case "proceed":
619 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
620 break;
621 case "retract":
622 receiveRetract(from, serverMessageId, timestamp);
623 break;
624 case "reject":
625 receiveReject(from, serverMessageId, timestamp);
626 break;
627 case "accept":
628 receiveAccept(from, serverMessageId, timestamp);
629 break;
630 default:
631 break;
632 }
633 }
634
635 void deliverFailedProceed() {
636 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
637 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
638 webRTCWrapper.close();
639 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
640 this.finish();
641 }
642 }
643
644 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
645 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
646 if (originatedFromMyself) {
647 if (transition(State.ACCEPTED)) {
648 if (serverMsgId != null) {
649 this.message.setServerMsgId(serverMsgId);
650 }
651 this.message.setTime(timestamp);
652 this.message.setCarbon(true); //indicate that call was accepted on other device
653 this.writeLogMessageSuccess(0);
654 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
655 this.finish();
656 } else {
657 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
658 }
659 } else {
660 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
661 }
662 }
663
664 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
665 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
666 //reject from another one of my clients
667 if (originatedFromMyself) {
668 receiveRejectFromMyself(serverMsgId, timestamp);
669 } else if (isInitiator()) {
670 if (from.equals(id.with)) {
671 receiveRejectFromResponder();
672 } else {
673 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
674 }
675 } else {
676 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
677 }
678 }
679
680 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
681 if (transition(State.REJECTED)) {
682 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
683 this.finish();
684 if (serverMsgId != null) {
685 this.message.setServerMsgId(serverMsgId);
686 }
687 this.message.setTime(timestamp);
688 this.message.setCarbon(true); //indicate that call was rejected on other device
689 writeLogMessageMissed();
690 } else {
691 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
692 }
693 }
694
695 private void receiveRejectFromResponder() {
696 if (isInState(State.PROCEED)) {
697 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
698 closeTransitionLogFinish(State.REJECTED_RACED);
699 return;
700 }
701 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
702 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
703 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
704 return;
705 }
706 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
707 }
708
709 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
710 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
711 if (originatedFromMyself) {
712 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
713 } else if (transition(State.PROPOSED, () -> {
714 final Collection<RtpDescription> descriptions = Collections2.transform(
715 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
716 input -> (RtpDescription) input
717 );
718 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
719 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
720 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
721 this.proposedMedia = Sets.newHashSet(media);
722 })) {
723 if (serverMsgId != null) {
724 this.message.setServerMsgId(serverMsgId);
725 }
726 this.message.setTime(timestamp);
727 startRinging();
728 } else {
729 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
730 }
731 }
732
733 private void startRinging() {
734 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
735 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
736 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
737 }
738
739 private synchronized void ringingTimeout() {
740 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
741 switch (this.state) {
742 case PROPOSED:
743 message.markUnread();
744 rejectCallFromProposed();
745 break;
746 case SESSION_INITIALIZED:
747 message.markUnread();
748 rejectCallFromSessionInitiate();
749 break;
750 }
751 }
752
753 private void cancelRingingTimeout() {
754 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
755 if (future != null && !future.isCancelled()) {
756 future.cancel(false);
757 }
758 }
759
760 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
761 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
762 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
763 if (from.equals(id.with)) {
764 if (isInitiator()) {
765 if (transition(State.PROCEED)) {
766 if (serverMsgId != null) {
767 this.message.setServerMsgId(serverMsgId);
768 }
769 this.message.setTime(timestamp);
770 final Integer remoteDeviceId = proceed.getDeviceId();
771 if (isOmemoEnabled()) {
772 this.omemoVerification.setDeviceId(remoteDeviceId);
773 } else {
774 if (remoteDeviceId != null) {
775 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
776 }
777 this.omemoVerification.setDeviceId(null);
778 }
779 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
780 } else {
781 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
782 }
783 } else {
784 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
785 }
786 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
787 if (transition(State.ACCEPTED)) {
788 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
789 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
790 this.finish();
791 }
792 } else {
793 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
794 }
795 }
796
797 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
798 if (from.equals(id.with)) {
799 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
800 if (transition(target)) {
801 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
802 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
803 if (serverMsgId != null) {
804 this.message.setServerMsgId(serverMsgId);
805 }
806 this.message.setTime(timestamp);
807 if (target == State.RETRACTED) {
808 this.message.markUnread();
809 }
810 writeLogMessageMissed();
811 finish();
812 } else {
813 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
814 }
815 } else {
816 //TODO parse retract from self
817 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
818 }
819 }
820
821 public void sendSessionInitiate() {
822 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
823 }
824
825 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
826 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
827 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
828 }
829
830 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
831 if (isTerminated()) {
832 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
833 return;
834 }
835 try {
836 setupWebRTC(media, iceServers);
837 } catch (final WebRTCWrapper.InitializationException e) {
838 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
839 webRTCWrapper.close();
840 sendRetract(Reason.ofThrowable(e));
841 return;
842 }
843 try {
844 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
845 prepareSessionInitiate(webRTCSessionDescription, targetState);
846 } catch (final Exception e) {
847 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
848 failureToInitiateSession(e, targetState);
849 }
850 }
851
852 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
853 if (isTerminated()) {
854 return;
855 }
856 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
857 webRTCWrapper.close();
858 final Reason reason = Reason.ofThrowable(throwable);
859 if (isInState(targetState)) {
860 sendSessionTerminate(reason);
861 } else {
862 sendRetract(reason);
863 }
864 }
865
866 private void sendRetract(final Reason reason) {
867 //TODO embed reason into retract
868 sendJingleMessage("retract", id.with.asBareJid());
869 transitionOrThrow(reasonToState(reason));
870 this.finish();
871 }
872
873 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
874 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
875 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
876 this.initiatorRtpContentMap = rtpContentMap;
877 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
878 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
879 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
880 @Override
881 public void onSuccess(final RtpContentMap outgoingContentMap) {
882 sendSessionInitiate(outgoingContentMap, targetState);
883 }
884
885 @Override
886 public void onFailure(@NonNull final Throwable throwable) {
887 failureToInitiateSession(throwable, targetState);
888 }
889 }, MoreExecutors.directExecutor());
890 }
891
892 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
893 if (isTerminated()) {
894 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
895 return;
896 }
897 this.transitionOrThrow(targetState);
898 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
899 send(sessionInitiate);
900 }
901
902 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
903 if (this.omemoVerification.hasDeviceId()) {
904 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
905 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
906 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
907 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
908 return verifiedPayload.getPayload();
909 }, MoreExecutors.directExecutor());
910 if (Config.REQUIRE_RTP_VERIFICATION) {
911 return future;
912 }
913 return Futures.catching(
914 future,
915 CryptoFailedException.class,
916 e -> {
917 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
918 return rtpContentMap;
919 },
920 MoreExecutors.directExecutor()
921 );
922 } else {
923 return Futures.immediateFuture(rtpContentMap);
924 }
925 }
926
927 private void sendSessionTerminate(final Reason reason) {
928 sendSessionTerminate(reason, null);
929 }
930
931 private void sendSessionTerminate(final Reason reason, final String text) {
932 final State previous = this.state;
933 final State target = reasonToState(reason);
934 transitionOrThrow(target);
935 if (previous != State.NULL) {
936 writeLogMessage(target);
937 }
938 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
939 jinglePacket.setReason(reason, text);
940 Log.d(Config.LOGTAG, jinglePacket.toString());
941 send(jinglePacket);
942 finish();
943 }
944
945 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
946 final RtpContentMap transportInfo;
947 try {
948 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
949 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
950 } catch (final Exception e) {
951 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
952 return;
953 }
954 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
955 send(jinglePacket);
956 }
957
958 private void send(final JinglePacket jinglePacket) {
959 jinglePacket.setTo(id.with);
960 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
961 }
962
963 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
964 if (response.getType() == IqPacket.TYPE.ERROR) {
965 final String errorCondition = response.getErrorCondition();
966 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
967 if (isTerminated()) {
968 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
969 return;
970 }
971 this.webRTCWrapper.close();
972 final State target;
973 if (Arrays.asList(
974 "service-unavailable",
975 "recipient-unavailable",
976 "remote-server-not-found",
977 "remote-server-timeout"
978 ).contains(errorCondition)) {
979 target = State.TERMINATED_CONNECTIVITY_ERROR;
980 } else {
981 target = State.TERMINATED_APPLICATION_FAILURE;
982 }
983 transitionOrThrow(target);
984 this.finish();
985 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
986 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
987 if (isTerminated()) {
988 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
989 return;
990 }
991 this.webRTCWrapper.close();
992 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
993 this.finish();
994 }
995 }
996
997 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
998 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
999 this.webRTCWrapper.close();
1000 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1001 respondWithOutOfOrder(jinglePacket);
1002 this.finish();
1003 }
1004
1005 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1006 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
1007 }
1008
1009 private void respondOk(final JinglePacket jinglePacket) {
1010 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1011 }
1012
1013 public void throwStateTransitionException() {
1014 final StateTransitionException exception = this.stateTransitionException;
1015 if (exception != null) {
1016 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1017 }
1018 }
1019
1020 public RtpEndUserState getEndUserState() {
1021 switch (this.state) {
1022 case NULL:
1023 case PROPOSED:
1024 case SESSION_INITIALIZED:
1025 if (isInitiator()) {
1026 return RtpEndUserState.RINGING;
1027 } else {
1028 return RtpEndUserState.INCOMING_CALL;
1029 }
1030 case PROCEED:
1031 if (isInitiator()) {
1032 return RtpEndUserState.RINGING;
1033 } else {
1034 return RtpEndUserState.ACCEPTING_CALL;
1035 }
1036 case SESSION_INITIALIZED_PRE_APPROVED:
1037 if (isInitiator()) {
1038 return RtpEndUserState.RINGING;
1039 } else {
1040 return RtpEndUserState.CONNECTING;
1041 }
1042 case SESSION_ACCEPTED:
1043 return getPeerConnectionStateAsEndUserState();
1044 case REJECTED:
1045 case REJECTED_RACED:
1046 case TERMINATED_DECLINED_OR_BUSY:
1047 if (isInitiator()) {
1048 return RtpEndUserState.DECLINED_OR_BUSY;
1049 } else {
1050 return RtpEndUserState.ENDED;
1051 }
1052 case TERMINATED_SUCCESS:
1053 case ACCEPTED:
1054 case RETRACTED:
1055 case TERMINATED_CANCEL_OR_TIMEOUT:
1056 return RtpEndUserState.ENDED;
1057 case RETRACTED_RACED:
1058 if (isInitiator()) {
1059 return RtpEndUserState.ENDED;
1060 } else {
1061 return RtpEndUserState.RETRACTED;
1062 }
1063 case TERMINATED_CONNECTIVITY_ERROR:
1064 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1065 case TERMINATED_APPLICATION_FAILURE:
1066 return RtpEndUserState.APPLICATION_ERROR;
1067 case TERMINATED_SECURITY_ERROR:
1068 return RtpEndUserState.SECURITY_ERROR;
1069 }
1070 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1071 }
1072
1073
1074 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1075 final PeerConnection.PeerConnectionState state;
1076 try {
1077 state = webRTCWrapper.getState();
1078 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1079 //We usually close the WebRTCWrapper *before* transitioning so we might still
1080 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1081 return RtpEndUserState.ENDING_CALL;
1082 }
1083 switch (state) {
1084 case CONNECTED:
1085 return RtpEndUserState.CONNECTED;
1086 case NEW:
1087 case CONNECTING:
1088 return RtpEndUserState.CONNECTING;
1089 case CLOSED:
1090 return RtpEndUserState.ENDING_CALL;
1091 default:
1092 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1093 }
1094 }
1095
1096 public Set<Media> getMedia() {
1097 final State current = getState();
1098 if (current == State.NULL) {
1099 if (isInitiator()) {
1100 return Preconditions.checkNotNull(
1101 this.proposedMedia,
1102 "RTP connection has not been initialized properly"
1103 );
1104 }
1105 throw new IllegalStateException("RTP connection has not been initialized yet");
1106 }
1107 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1108 return Preconditions.checkNotNull(
1109 this.proposedMedia,
1110 "RTP connection has not been initialized properly"
1111 );
1112 }
1113 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1114 if (initiatorContentMap != null) {
1115 return initiatorContentMap.getMedia();
1116 } else if (isTerminated()) {
1117 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1118 } else {
1119 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1120 }
1121 }
1122
1123
1124 public boolean isVerified() {
1125 final String fingerprint = this.omemoVerification.getFingerprint();
1126 if (fingerprint == null) {
1127 return false;
1128 }
1129 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1130 return status != null && status.isVerified();
1131 }
1132
1133 public synchronized void acceptCall() {
1134 switch (this.state) {
1135 case PROPOSED:
1136 cancelRingingTimeout();
1137 acceptCallFromProposed();
1138 break;
1139 case SESSION_INITIALIZED:
1140 cancelRingingTimeout();
1141 acceptCallFromSessionInitialized();
1142 break;
1143 case ACCEPTED:
1144 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1145 break;
1146 case PROCEED:
1147 case SESSION_ACCEPTED:
1148 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1149 break;
1150 default:
1151 throw new IllegalStateException("Can not accept call from " + this.state);
1152 }
1153 }
1154
1155
1156 public void notifyPhoneCall() {
1157 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1158 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1159 rejectCall();
1160 } else {
1161 endCall();
1162 }
1163 }
1164
1165 public synchronized void rejectCall() {
1166 if (isTerminated()) {
1167 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1168 return;
1169 }
1170 switch (this.state) {
1171 case PROPOSED:
1172 rejectCallFromProposed();
1173 break;
1174 case SESSION_INITIALIZED:
1175 rejectCallFromSessionInitiate();
1176 break;
1177 default:
1178 throw new IllegalStateException("Can not reject call from " + this.state);
1179 }
1180 }
1181
1182 public synchronized void endCall() {
1183 if (isTerminated()) {
1184 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1185 return;
1186 }
1187 if (isInState(State.PROPOSED) && !isInitiator()) {
1188 rejectCallFromProposed();
1189 return;
1190 }
1191 if (isInState(State.PROCEED)) {
1192 if (isInitiator()) {
1193 retractFromProceed();
1194 } else {
1195 rejectCallFromProceed();
1196 }
1197 return;
1198 }
1199 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1200 this.webRTCWrapper.close();
1201 sendSessionTerminate(Reason.CANCEL);
1202 return;
1203 }
1204 if (isInState(State.SESSION_INITIALIZED)) {
1205 rejectCallFromSessionInitiate();
1206 return;
1207 }
1208 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1209 this.webRTCWrapper.close();
1210 sendSessionTerminate(Reason.SUCCESS);
1211 return;
1212 }
1213 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1214 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1215 return;
1216 }
1217 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1218 }
1219
1220 private void retractFromProceed() {
1221 Log.d(Config.LOGTAG, "retract from proceed");
1222 this.sendJingleMessage("retract");
1223 closeTransitionLogFinish(State.RETRACTED_RACED);
1224 }
1225
1226 private void closeTransitionLogFinish(final State state) {
1227 this.webRTCWrapper.close();
1228 transitionOrThrow(state);
1229 writeLogMessage(state);
1230 finish();
1231 }
1232
1233 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1234 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1235 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1236 if (media.contains(Media.VIDEO)) {
1237 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1238 } else {
1239 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1240 }
1241 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1242 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1243 }
1244
1245 private void acceptCallFromProposed() {
1246 transitionOrThrow(State.PROCEED);
1247 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1248 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1249 this.sendJingleMessage("proceed");
1250 }
1251
1252 private void rejectCallFromProposed() {
1253 transitionOrThrow(State.REJECTED);
1254 writeLogMessageMissed();
1255 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1256 this.sendJingleMessage("reject");
1257 finish();
1258 }
1259
1260 private void rejectCallFromProceed() {
1261 this.sendJingleMessage("reject");
1262 closeTransitionLogFinish(State.REJECTED_RACED);
1263 }
1264
1265 private void rejectCallFromSessionInitiate() {
1266 webRTCWrapper.close();
1267 sendSessionTerminate(Reason.DECLINE);
1268 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1269 }
1270
1271 private void sendJingleMessage(final String action) {
1272 sendJingleMessage(action, id.with);
1273 }
1274
1275 private void sendJingleMessage(final String action, final Jid to) {
1276 final MessagePacket messagePacket = new MessagePacket();
1277 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1278 messagePacket.setTo(to);
1279 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1280 if ("proceed".equals(action)) {
1281 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1282 if (isOmemoEnabled()) {
1283 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1284 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1285 device.setAttribute("id", deviceId);
1286 }
1287 }
1288 messagePacket.addChild("store", "urn:xmpp:hints");
1289 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1290 }
1291
1292 private boolean isOmemoEnabled() {
1293 final Conversational conversational = message.getConversation();
1294 if (conversational instanceof Conversation) {
1295 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1296 }
1297 return false;
1298 }
1299
1300 private void acceptCallFromSessionInitialized() {
1301 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1302 sendSessionAccept();
1303 }
1304
1305 private synchronized boolean isInState(State... state) {
1306 return Arrays.asList(state).contains(this.state);
1307 }
1308
1309 private boolean transition(final State target) {
1310 return transition(target, null);
1311 }
1312
1313 private synchronized boolean transition(final State target, final Runnable runnable) {
1314 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1315 if (validTransitions != null && validTransitions.contains(target)) {
1316 this.state = target;
1317 this.stateTransitionException = new StateTransitionException(target);
1318 if (runnable != null) {
1319 runnable.run();
1320 }
1321 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1322 updateEndUserState();
1323 updateOngoingCallNotification();
1324 return true;
1325 } else {
1326 return false;
1327 }
1328 }
1329
1330 void transitionOrThrow(final State target) {
1331 if (!transition(target)) {
1332 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1333 }
1334 }
1335
1336 @Override
1337 public void onIceCandidate(final IceCandidate iceCandidate) {
1338 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1339 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1340 sendTransportInfo(iceCandidate.sdpMid, candidate);
1341 }
1342
1343 @Override
1344 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1345 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1346 this.stateHistory.add(newState);
1347 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1348 this.sessionDuration.start();
1349 } else if (this.sessionDuration.isRunning()) {
1350 this.sessionDuration.stop();
1351 }
1352
1353 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1354 final boolean failedOrDisconnected = Arrays.asList(
1355 PeerConnection.PeerConnectionState.FAILED,
1356 PeerConnection.PeerConnectionState.DISCONNECTED
1357 ).contains(newState);
1358
1359
1360 if (neverConnected && failedOrDisconnected) {
1361 if (isTerminated()) {
1362 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1363 return;
1364 }
1365 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1366 } else if (newState == PeerConnection.PeerConnectionState.FAILED) {
1367 Log.d(Config.LOGTAG, "attempting to restart ICE");
1368 webRTCWrapper.restartIce();
1369 }
1370 updateEndUserState();
1371 }
1372
1373 @Override
1374 public void onRenegotiationNeeded() {
1375 Log.d(Config.LOGTAG, "onRenegotiationNeeded()");
1376 this.webRTCWrapper.execute(this::renegotiate);
1377 }
1378
1379 private void renegotiate() {
1380 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1381 try {
1382 final SessionDescription sessionDescription = setLocalSessionDescription();
1383 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1384 setRenegotiatedContentMap(rtpContentMap);
1385 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1386 } catch (final Exception e) {
1387 Log.d(Config.LOGTAG, "failed to renegotiate", e);
1388 //TODO send some sort of failure (comparable to when initiating)
1389 }
1390 }
1391
1392 private void setRenegotiatedContentMap(final RtpContentMap rtpContentMap) {
1393 if (isInitiator()) {
1394 this.initiatorRtpContentMap = rtpContentMap;
1395 } else {
1396 this.responderRtpContentMap = rtpContentMap;
1397 }
1398 }
1399
1400 private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException {
1401 final org.webrtc.SessionDescription sessionDescription = this.webRTCWrapper.setLocalDescription().get();
1402 return SessionDescription.parse(sessionDescription.description);
1403 }
1404
1405 private void closeWebRTCSessionAfterFailedConnection() {
1406 this.webRTCWrapper.close();
1407 synchronized (this) {
1408 if (isTerminated()) {
1409 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1410 return;
1411 }
1412 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1413 }
1414 }
1415
1416 public boolean zeroDuration() {
1417 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1418 }
1419
1420 public long getCallDuration() {
1421 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1422 }
1423
1424 public AppRTCAudioManager getAudioManager() {
1425 return webRTCWrapper.getAudioManager();
1426 }
1427
1428 public boolean isMicrophoneEnabled() {
1429 return webRTCWrapper.isMicrophoneEnabled();
1430 }
1431
1432 public boolean setMicrophoneEnabled(final boolean enabled) {
1433 return webRTCWrapper.setMicrophoneEnabled(enabled);
1434 }
1435
1436 public boolean isVideoEnabled() {
1437 return webRTCWrapper.isVideoEnabled();
1438 }
1439
1440 public void setVideoEnabled(final boolean enabled) {
1441 webRTCWrapper.setVideoEnabled(enabled);
1442 }
1443
1444 public boolean isCameraSwitchable() {
1445 return webRTCWrapper.isCameraSwitchable();
1446 }
1447
1448 public boolean isFrontCamera() {
1449 return webRTCWrapper.isFrontCamera();
1450 }
1451
1452 public ListenableFuture<Boolean> switchCamera() {
1453 return webRTCWrapper.switchCamera();
1454 }
1455
1456 @Override
1457 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1458 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1459 }
1460
1461 private void updateEndUserState() {
1462 final RtpEndUserState endUserState = getEndUserState();
1463 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1464 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1465 }
1466
1467 private void updateOngoingCallNotification() {
1468 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1469 xmppConnectionService.setOngoingCall(id, getMedia());
1470 } else {
1471 xmppConnectionService.removeOngoingCall();
1472 }
1473 }
1474
1475 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1476 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1477 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1478 request.setTo(id.account.getDomain());
1479 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1480 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1481 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1482 if (response.getType() == IqPacket.TYPE.RESULT) {
1483 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1484 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1485 for (final Element child : children) {
1486 if ("service".equals(child.getName())) {
1487 final String type = child.getAttribute("type");
1488 final String host = child.getAttribute("host");
1489 final String sport = child.getAttribute("port");
1490 final Integer port = sport == null ? null : Ints.tryParse(sport);
1491 final String transport = child.getAttribute("transport");
1492 final String username = child.getAttribute("username");
1493 final String password = child.getAttribute("password");
1494 if (Strings.isNullOrEmpty(host) || port == null) {
1495 continue;
1496 }
1497 if (port < 0 || port > 65535) {
1498 continue;
1499 }
1500 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1501 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1502 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1503 continue;
1504 }
1505 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1506 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1507 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1508 if (username != null && password != null) {
1509 iceServerBuilder.setUsername(username);
1510 iceServerBuilder.setPassword(password);
1511 } else if (Arrays.asList("turn", "turns").contains(type)) {
1512 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1513 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1514 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1515 continue;
1516 }
1517 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1518 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1519 listBuilder.add(iceServer);
1520 }
1521 }
1522 }
1523 }
1524 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1525 if (iceServers.size() == 0) {
1526 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1527 }
1528 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1529 });
1530 } else {
1531 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1532 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1533 }
1534 }
1535
1536 private void finish() {
1537 if (isTerminated()) {
1538 this.cancelRingingTimeout();
1539 this.webRTCWrapper.verifyClosed();
1540 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1541 this.jingleConnectionManager.finishConnectionOrThrow(this);
1542 } else {
1543 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1544 }
1545 }
1546
1547 private void writeLogMessage(final State state) {
1548 final long duration = getCallDuration();
1549 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1550 writeLogMessageSuccess(duration);
1551 } else {
1552 writeLogMessageMissed();
1553 }
1554 }
1555
1556 private void writeLogMessageSuccess(final long duration) {
1557 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1558 this.writeMessage();
1559 }
1560
1561 private void writeLogMessageMissed() {
1562 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1563 this.writeMessage();
1564 }
1565
1566 private void writeMessage() {
1567 final Conversational conversational = message.getConversation();
1568 if (conversational instanceof Conversation) {
1569 ((Conversation) conversational).add(this.message);
1570 xmppConnectionService.createMessageAsync(message);
1571 xmppConnectionService.updateConversationUi();
1572 } else {
1573 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1574 }
1575 }
1576
1577 public State getState() {
1578 return this.state;
1579 }
1580
1581 boolean isTerminated() {
1582 return TERMINATED.contains(this.state);
1583 }
1584
1585 public Optional<VideoTrack> getLocalVideoTrack() {
1586 return webRTCWrapper.getLocalVideoTrack();
1587 }
1588
1589 public Optional<VideoTrack> getRemoteVideoTrack() {
1590 return webRTCWrapper.getRemoteVideoTrack();
1591 }
1592
1593
1594 public EglBase.Context getEglBaseContext() {
1595 return webRTCWrapper.getEglBaseContext();
1596 }
1597
1598 void setProposedMedia(final Set<Media> media) {
1599 this.proposedMedia = media;
1600 }
1601
1602 public void fireStateUpdate() {
1603 final RtpEndUserState endUserState = getEndUserState();
1604 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1605 }
1606
1607 private interface OnIceServersDiscovered {
1608 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1609 }
1610
1611 private static class StateTransitionException extends Exception {
1612 private final State state;
1613
1614 private StateTransitionException(final State state) {
1615 this.state = state;
1616 }
1617 }
1618}