1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Predicates;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.Maps;
18import com.google.common.collect.Sets;
19import com.google.common.primitives.Ints;
20import com.google.common.util.concurrent.FutureCallback;
21import com.google.common.util.concurrent.Futures;
22import com.google.common.util.concurrent.ListenableFuture;
23import com.google.common.util.concurrent.MoreExecutors;
24
25import org.webrtc.EglBase;
26import org.webrtc.IceCandidate;
27import org.webrtc.PeerConnection;
28import org.webrtc.VideoTrack;
29
30import java.util.Arrays;
31import java.util.Collection;
32import java.util.Collections;
33import java.util.LinkedList;
34import java.util.List;
35import java.util.Map;
36import java.util.Queue;
37import java.util.Set;
38import java.util.concurrent.ExecutionException;
39import java.util.concurrent.ScheduledFuture;
40import java.util.concurrent.TimeUnit;
41
42import eu.siacs.conversations.Config;
43import eu.siacs.conversations.crypto.axolotl.AxolotlService;
44import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
45import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
46import eu.siacs.conversations.entities.Account;
47import eu.siacs.conversations.entities.Conversation;
48import eu.siacs.conversations.entities.Conversational;
49import eu.siacs.conversations.entities.Message;
50import eu.siacs.conversations.entities.RtpSessionStatus;
51import eu.siacs.conversations.services.AppRTCAudioManager;
52import eu.siacs.conversations.utils.IP;
53import eu.siacs.conversations.xml.Element;
54import eu.siacs.conversations.xml.Namespace;
55import eu.siacs.conversations.xmpp.Jid;
56import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
57import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
58import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
61import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
62import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
63import eu.siacs.conversations.xmpp.stanzas.IqPacket;
64import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
65
66public class JingleRtpConnection extends AbstractJingleConnection
67 implements WebRTCWrapper.EventCallback {
68
69 public static final List<State> STATES_SHOWING_ONGOING_CALL =
70 Arrays.asList(
71 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
72 private static final long BUSY_TIME_OUT = 30;
73 private static final List<State> TERMINATED =
74 Arrays.asList(
75 State.ACCEPTED,
76 State.REJECTED,
77 State.REJECTED_RACED,
78 State.RETRACTED,
79 State.RETRACTED_RACED,
80 State.TERMINATED_SUCCESS,
81 State.TERMINATED_DECLINED_OR_BUSY,
82 State.TERMINATED_CONNECTIVITY_ERROR,
83 State.TERMINATED_CANCEL_OR_TIMEOUT,
84 State.TERMINATED_APPLICATION_FAILURE,
85 State.TERMINATED_SECURITY_ERROR);
86
87 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
88
89 static {
90 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
91 new ImmutableMap.Builder<>();
92 transitionBuilder.put(
93 State.NULL,
94 ImmutableList.of(
95 State.PROPOSED,
96 State.SESSION_INITIALIZED,
97 State.TERMINATED_APPLICATION_FAILURE,
98 State.TERMINATED_SECURITY_ERROR));
99 transitionBuilder.put(
100 State.PROPOSED,
101 ImmutableList.of(
102 State.ACCEPTED,
103 State.PROCEED,
104 State.REJECTED,
105 State.RETRACTED,
106 State.TERMINATED_APPLICATION_FAILURE,
107 State.TERMINATED_SECURITY_ERROR,
108 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
109 // rebinds
110 ));
111 transitionBuilder.put(
112 State.PROCEED,
113 ImmutableList.of(
114 State.REJECTED_RACED,
115 State.RETRACTED_RACED,
116 State.SESSION_INITIALIZED_PRE_APPROVED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_APPLICATION_FAILURE,
119 State.TERMINATED_SECURITY_ERROR,
120 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
121 // bounces of the proceed message
122 ));
123 transitionBuilder.put(
124 State.SESSION_INITIALIZED,
125 ImmutableList.of(
126 State.SESSION_ACCEPTED,
127 State.TERMINATED_SUCCESS,
128 State.TERMINATED_DECLINED_OR_BUSY,
129 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
130 // and IQ timeouts
131 State.TERMINATED_CANCEL_OR_TIMEOUT,
132 State.TERMINATED_APPLICATION_FAILURE,
133 State.TERMINATED_SECURITY_ERROR));
134 transitionBuilder.put(
135 State.SESSION_INITIALIZED_PRE_APPROVED,
136 ImmutableList.of(
137 State.SESSION_ACCEPTED,
138 State.TERMINATED_SUCCESS,
139 State.TERMINATED_DECLINED_OR_BUSY,
140 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
141 // and IQ timeouts
142 State.TERMINATED_CANCEL_OR_TIMEOUT,
143 State.TERMINATED_APPLICATION_FAILURE,
144 State.TERMINATED_SECURITY_ERROR));
145 transitionBuilder.put(
146 State.SESSION_ACCEPTED,
147 ImmutableList.of(
148 State.TERMINATED_SUCCESS,
149 State.TERMINATED_DECLINED_OR_BUSY,
150 State.TERMINATED_CONNECTIVITY_ERROR,
151 State.TERMINATED_CANCEL_OR_TIMEOUT,
152 State.TERMINATED_APPLICATION_FAILURE,
153 State.TERMINATED_SECURITY_ERROR));
154 VALID_TRANSITIONS = transitionBuilder.build();
155 }
156
157 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
158 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
159 pendingIceCandidates = new LinkedList<>();
160 private final OmemoVerification omemoVerification = new OmemoVerification();
161 private final Message message;
162 private State state = State.NULL;
163 private Set<Media> proposedMedia;
164 private RtpContentMap initiatorRtpContentMap;
165 private RtpContentMap responderRtpContentMap;
166 private IceUdpTransportInfo.Setup peerDtlsSetup;
167 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
168 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
169 private ScheduledFuture<?> ringingTimeoutFuture;
170
171 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
172 super(jingleConnectionManager, id, initiator);
173 final Conversation conversation =
174 jingleConnectionManager
175 .getXmppConnectionService()
176 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
177 this.message =
178 new Message(
179 conversation,
180 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
181 Message.TYPE_RTP_SESSION,
182 id.sessionId);
183 }
184
185 private static State reasonToState(Reason reason) {
186 switch (reason) {
187 case SUCCESS:
188 return State.TERMINATED_SUCCESS;
189 case DECLINE:
190 case BUSY:
191 return State.TERMINATED_DECLINED_OR_BUSY;
192 case CANCEL:
193 case TIMEOUT:
194 return State.TERMINATED_CANCEL_OR_TIMEOUT;
195 case SECURITY_ERROR:
196 return State.TERMINATED_SECURITY_ERROR;
197 case FAILED_APPLICATION:
198 case UNSUPPORTED_TRANSPORTS:
199 case UNSUPPORTED_APPLICATIONS:
200 return State.TERMINATED_APPLICATION_FAILURE;
201 default:
202 return State.TERMINATED_CONNECTIVITY_ERROR;
203 }
204 }
205
206 @Override
207 synchronized void deliverPacket(final JinglePacket jinglePacket) {
208 switch (jinglePacket.getAction()) {
209 case SESSION_INITIATE:
210 receiveSessionInitiate(jinglePacket);
211 break;
212 case TRANSPORT_INFO:
213 receiveTransportInfo(jinglePacket);
214 break;
215 case SESSION_ACCEPT:
216 receiveSessionAccept(jinglePacket);
217 break;
218 case SESSION_TERMINATE:
219 receiveSessionTerminate(jinglePacket);
220 break;
221 default:
222 respondOk(jinglePacket);
223 Log.d(
224 Config.LOGTAG,
225 String.format(
226 "%s: received unhandled jingle action %s",
227 id.account.getJid().asBareJid(), jinglePacket.getAction()));
228 break;
229 }
230 }
231
232 @Override
233 synchronized void notifyRebound() {
234 if (isTerminated()) {
235 return;
236 }
237 webRTCWrapper.close();
238 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
239 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
240 }
241 if (isInState(
242 State.SESSION_INITIALIZED,
243 State.SESSION_INITIALIZED_PRE_APPROVED,
244 State.SESSION_ACCEPTED)) {
245 // we might have already changed resources (full jid) at this point; so this might not
246 // even reach the other party
247 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
248 } else {
249 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
250 finish();
251 }
252 }
253
254 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
255 respondOk(jinglePacket);
256 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
257 final State previous = this.state;
258 Log.d(
259 Config.LOGTAG,
260 id.account.getJid().asBareJid()
261 + ": received session terminate reason="
262 + wrapper.reason
263 + "("
264 + Strings.nullToEmpty(wrapper.text)
265 + ") while in state "
266 + previous);
267 if (TERMINATED.contains(previous)) {
268 Log.d(
269 Config.LOGTAG,
270 id.account.getJid().asBareJid()
271 + ": ignoring session terminate because already in "
272 + previous);
273 return;
274 }
275 webRTCWrapper.close();
276 final State target = reasonToState(wrapper.reason);
277 transitionOrThrow(target);
278 writeLogMessage(target);
279 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
280 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
281 }
282 finish();
283 }
284
285 private void receiveTransportInfo(final JinglePacket jinglePacket) {
286 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
287 // INITIALIZED only after transport-info has been received
288 if (isInState(
289 State.NULL,
290 State.PROCEED,
291 State.SESSION_INITIALIZED,
292 State.SESSION_INITIALIZED_PRE_APPROVED,
293 State.SESSION_ACCEPTED)) {
294 final RtpContentMap contentMap;
295 try {
296 contentMap = RtpContentMap.of(jinglePacket);
297 } catch (final IllegalArgumentException | NullPointerException e) {
298 Log.d(
299 Config.LOGTAG,
300 id.account.getJid().asBareJid()
301 + ": improperly formatted contents; ignoring",
302 e);
303 respondOk(jinglePacket);
304 return;
305 }
306 receiveTransportInfo(jinglePacket, contentMap);
307 } else {
308 if (isTerminated()) {
309 respondOk(jinglePacket);
310 Log.d(
311 Config.LOGTAG,
312 id.account.getJid().asBareJid()
313 + ": ignoring out-of-order transport info; we where already terminated");
314 } else {
315 Log.d(
316 Config.LOGTAG,
317 id.account.getJid().asBareJid()
318 + ": received transport info while in state="
319 + this.state);
320 terminateWithOutOfOrder(jinglePacket);
321 }
322 }
323 }
324
325 private void receiveTransportInfo(
326 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
327 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
328 contentMap.contents.entrySet();
329 if (this.state == State.SESSION_ACCEPTED) {
330 // zero candidates + modified credentials are an ICE restart offer
331 if (checkForIceRestart(jinglePacket, contentMap)) {
332 return;
333 }
334 respondOk(jinglePacket);
335 try {
336 processCandidates(candidates);
337 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
338 Log.w(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
342 }
343 } else {
344 respondOk(jinglePacket);
345 pendingIceCandidates.addAll(candidates);
346 }
347 }
348
349 private boolean checkForIceRestart(
350 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
351 final RtpContentMap existing = getRemoteContentMap();
352 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
353 final IceUdpTransportInfo.Credentials newCredentials;
354 try {
355 existingCredentials = existing.getCredentials();
356 newCredentials = rtpContentMap.getDistinctCredentials();
357 } catch (final IllegalStateException e) {
358 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
359 return false;
360 }
361 if (existingCredentials.contains(newCredentials)) {
362 return false;
363 }
364 // TODO an alternative approach is to check if we already got an iq result to our
365 // ICE-restart
366 // and if that's the case we are seeing an answer.
367 // This might be more spec compliant but also more error prone potentially
368 final boolean isOffer = rtpContentMap.emptyCandidates();
369 final RtpContentMap restartContentMap;
370 try {
371 if (isOffer) {
372 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
373 restartContentMap =
374 existing.modifiedCredentials(
375 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
376 } else {
377 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
378 Log.d(
379 Config.LOGTAG,
380 "received confirmation of ICE restart"
381 + newCredentials
382 + " peer_setup="
383 + setup);
384 // DTLS setup attribute needs to be rewritten to reflect current peer state
385 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
386 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
387 }
388 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
389 return isOffer;
390 } else {
391 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
392 respondWithTieBreak(jinglePacket);
393 return true;
394 }
395 } catch (final Exception exception) {
396 respondOk(jinglePacket);
397 final Throwable rootCause = Throwables.getRootCause(exception);
398 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
399 // If this happens a termination is already in progress
400 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
401 return true;
402 }
403 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
404 webRTCWrapper.close();
405 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
406 return true;
407 }
408 }
409
410 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
411 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
412 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
413 throw new IllegalStateException("Invalid peer setup");
414 }
415 return peerSetup;
416 }
417
418 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
419 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
420 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
421 }
422 this.peerDtlsSetup = setup;
423 }
424
425 private boolean applyIceRestart(
426 final JinglePacket jinglePacket,
427 final RtpContentMap restartContentMap,
428 final boolean isOffer)
429 throws ExecutionException, InterruptedException {
430 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
431 final org.webrtc.SessionDescription.Type type =
432 isOffer
433 ? org.webrtc.SessionDescription.Type.OFFER
434 : org.webrtc.SessionDescription.Type.ANSWER;
435 org.webrtc.SessionDescription sdp =
436 new org.webrtc.SessionDescription(type, sessionDescription.toString());
437 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
438 if (isInitiator()) {
439 // We ignore the offer and respond with tie-break. This will clause the responder
440 // not to apply the content map
441 return false;
442 }
443 }
444 webRTCWrapper.setRemoteDescription(sdp).get();
445 setRemoteContentMap(restartContentMap);
446 if (isOffer) {
447 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
448 final SessionDescription localSessionDescription = setLocalSessionDescription();
449 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
450 // We need to respond OK before sending any candidates
451 respondOk(jinglePacket);
452 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
453 } else {
454 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
455 }
456 return true;
457 }
458
459 private void processCandidates(
460 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
461 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
462 processCandidate(content);
463 }
464 }
465
466 private void processCandidate(
467 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
468 final RtpContentMap rtpContentMap = getRemoteContentMap();
469 final List<String> indices = toIdentificationTags(rtpContentMap);
470 final String sdpMid = content.getKey(); // aka content name
471 final IceUdpTransportInfo transport = content.getValue().transport;
472 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
473
474 // TODO check that credentials remained the same
475
476 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
477 final String sdp;
478 try {
479 sdp = candidate.toSdpAttribute(credentials.ufrag);
480 } catch (final IllegalArgumentException e) {
481 Log.d(
482 Config.LOGTAG,
483 id.account.getJid().asBareJid()
484 + ": ignoring invalid ICE candidate "
485 + e.getMessage());
486 continue;
487 }
488 final int mLineIndex = indices.indexOf(sdpMid);
489 if (mLineIndex < 0) {
490 Log.w(
491 Config.LOGTAG,
492 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
493 }
494 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
495 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
496 this.webRTCWrapper.addIceCandidate(iceCandidate);
497 }
498 }
499
500 private RtpContentMap getRemoteContentMap() {
501 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
502 }
503
504 private RtpContentMap getLocalContentMap() {
505 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
506 }
507
508 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
509 final Group originalGroup = rtpContentMap.group;
510 final List<String> identificationTags =
511 originalGroup == null
512 ? rtpContentMap.getNames()
513 : originalGroup.getIdentificationTags();
514 if (identificationTags.size() == 0) {
515 Log.w(
516 Config.LOGTAG,
517 id.account.getJid().asBareJid()
518 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
519 }
520 return identificationTags;
521 }
522
523 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
524 final JinglePacket jinglePacket, final boolean expectVerification) {
525 final RtpContentMap receivedContentMap;
526 try {
527 receivedContentMap = RtpContentMap.of(jinglePacket);
528 } catch (final Exception e) {
529 return Futures.immediateFailedFuture(e);
530 }
531 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
532 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
533 id.account
534 .getAxolotlService()
535 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
536 return Futures.transform(
537 future,
538 omemoVerifiedPayload -> {
539 // TODO test if an exception here triggers a correct abort
540 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
541 Log.d(
542 Config.LOGTAG,
543 id.account.getJid().asBareJid()
544 + ": received verifiable DTLS fingerprint via "
545 + omemoVerification);
546 return omemoVerifiedPayload.getPayload();
547 },
548 MoreExecutors.directExecutor());
549 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
550 return Futures.immediateFailedFuture(
551 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
552 } else {
553 return Futures.immediateFuture(receivedContentMap);
554 }
555 }
556
557 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
558 if (isInitiator()) {
559 Log.d(
560 Config.LOGTAG,
561 String.format(
562 "%s: received session-initiate even though we were initiating",
563 id.account.getJid().asBareJid()));
564 if (isTerminated()) {
565 Log.d(
566 Config.LOGTAG,
567 String.format(
568 "%s: got a reason to terminate with out-of-order. but already in state %s",
569 id.account.getJid().asBareJid(), getState()));
570 respondWithOutOfOrder(jinglePacket);
571 } else {
572 terminateWithOutOfOrder(jinglePacket);
573 }
574 return;
575 }
576 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
577 Futures.addCallback(
578 future,
579 new FutureCallback<RtpContentMap>() {
580 @Override
581 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
582 receiveSessionInitiate(jinglePacket, rtpContentMap);
583 }
584
585 @Override
586 public void onFailure(@NonNull final Throwable throwable) {
587 respondOk(jinglePacket);
588 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
589 }
590 },
591 MoreExecutors.directExecutor());
592 }
593
594 private void receiveSessionInitiate(
595 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
596 try {
597 contentMap.requireContentDescriptions();
598 contentMap.requireDTLSFingerprint(true);
599 } catch (final RuntimeException e) {
600 Log.d(
601 Config.LOGTAG,
602 id.account.getJid().asBareJid() + ": improperly formatted contents",
603 Throwables.getRootCause(e));
604 respondOk(jinglePacket);
605 sendSessionTerminate(Reason.of(e), e.getMessage());
606 return;
607 }
608 Log.d(
609 Config.LOGTAG,
610 "processing session-init with " + contentMap.contents.size() + " contents");
611 final State target;
612 if (this.state == State.PROCEED) {
613 Preconditions.checkState(
614 proposedMedia != null && proposedMedia.size() > 0,
615 "proposed media must be set when processing pre-approved session-initiate");
616 if (!this.proposedMedia.equals(contentMap.getMedia())) {
617 sendSessionTerminate(
618 Reason.SECURITY_ERROR,
619 String.format(
620 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
621 this.proposedMedia, contentMap.getMedia()));
622 return;
623 }
624 target = State.SESSION_INITIALIZED_PRE_APPROVED;
625 } else {
626 target = State.SESSION_INITIALIZED;
627 }
628 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
629 respondOk(jinglePacket);
630 pendingIceCandidates.addAll(contentMap.contents.entrySet());
631 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
632 Log.d(
633 Config.LOGTAG,
634 id.account.getJid().asBareJid()
635 + ": automatically accepting session-initiate");
636 sendSessionAccept();
637 } else {
638 Log.d(
639 Config.LOGTAG,
640 id.account.getJid().asBareJid()
641 + ": received not pre-approved session-initiate. start ringing");
642 startRinging();
643 }
644 } else {
645 Log.d(
646 Config.LOGTAG,
647 String.format(
648 "%s: received session-initiate while in state %s",
649 id.account.getJid().asBareJid(), state));
650 terminateWithOutOfOrder(jinglePacket);
651 }
652 }
653
654 private void receiveSessionAccept(final JinglePacket jinglePacket) {
655 if (!isInitiator()) {
656 Log.d(
657 Config.LOGTAG,
658 String.format(
659 "%s: received session-accept even though we were responding",
660 id.account.getJid().asBareJid()));
661 terminateWithOutOfOrder(jinglePacket);
662 return;
663 }
664 final ListenableFuture<RtpContentMap> future =
665 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
666 Futures.addCallback(
667 future,
668 new FutureCallback<RtpContentMap>() {
669 @Override
670 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
671 receiveSessionAccept(jinglePacket, rtpContentMap);
672 }
673
674 @Override
675 public void onFailure(@NonNull final Throwable throwable) {
676 respondOk(jinglePacket);
677 Log.d(
678 Config.LOGTAG,
679 id.account.getJid().asBareJid()
680 + ": improperly formatted contents in session-accept",
681 throwable);
682 webRTCWrapper.close();
683 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
684 }
685 },
686 MoreExecutors.directExecutor());
687 }
688
689 private void receiveSessionAccept(
690 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
691 try {
692 contentMap.requireContentDescriptions();
693 contentMap.requireDTLSFingerprint();
694 } catch (final RuntimeException e) {
695 respondOk(jinglePacket);
696 Log.d(
697 Config.LOGTAG,
698 id.account.getJid().asBareJid()
699 + ": improperly formatted contents in session-accept",
700 e);
701 webRTCWrapper.close();
702 sendSessionTerminate(Reason.of(e), e.getMessage());
703 return;
704 }
705 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
706 if (!initiatorMedia.equals(contentMap.getMedia())) {
707 sendSessionTerminate(
708 Reason.SECURITY_ERROR,
709 String.format(
710 "Your session-included included media %s but our session-initiate was %s",
711 this.proposedMedia, contentMap.getMedia()));
712 return;
713 }
714 Log.d(
715 Config.LOGTAG,
716 "processing session-accept with " + contentMap.contents.size() + " contents");
717 if (transition(State.SESSION_ACCEPTED)) {
718 respondOk(jinglePacket);
719 receiveSessionAccept(contentMap);
720 } else {
721 Log.d(
722 Config.LOGTAG,
723 String.format(
724 "%s: received session-accept while in state %s",
725 id.account.getJid().asBareJid(), state));
726 respondOk(jinglePacket);
727 }
728 }
729
730 private void receiveSessionAccept(final RtpContentMap contentMap) {
731 this.responderRtpContentMap = contentMap;
732 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
733 final SessionDescription sessionDescription;
734 try {
735 sessionDescription = SessionDescription.of(contentMap, false);
736 } catch (final IllegalArgumentException | NullPointerException e) {
737 Log.d(
738 Config.LOGTAG,
739 id.account.getJid().asBareJid()
740 + ": unable convert offer from session-accept to SDP",
741 e);
742 webRTCWrapper.close();
743 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
744 return;
745 }
746 final org.webrtc.SessionDescription answer =
747 new org.webrtc.SessionDescription(
748 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
749 try {
750 this.webRTCWrapper.setRemoteDescription(answer).get();
751 } catch (final Exception e) {
752 Log.d(
753 Config.LOGTAG,
754 id.account.getJid().asBareJid()
755 + ": unable to set remote description after receiving session-accept",
756 Throwables.getRootCause(e));
757 webRTCWrapper.close();
758 sendSessionTerminate(
759 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
760 return;
761 }
762 processCandidates(contentMap.contents.entrySet());
763 }
764
765 private void sendSessionAccept() {
766 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
767 if (rtpContentMap == null) {
768 throw new IllegalStateException("initiator RTP Content Map has not been set");
769 }
770 final SessionDescription offer;
771 try {
772 offer = SessionDescription.of(rtpContentMap, true);
773 } catch (final IllegalArgumentException | NullPointerException e) {
774 Log.d(
775 Config.LOGTAG,
776 id.account.getJid().asBareJid()
777 + ": unable convert offer from session-initiate to SDP",
778 e);
779 webRTCWrapper.close();
780 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
781 return;
782 }
783 sendSessionAccept(rtpContentMap.getMedia(), offer);
784 }
785
786 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
787 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
788 }
789
790 private synchronized void sendSessionAccept(
791 final Set<Media> media,
792 final SessionDescription offer,
793 final List<PeerConnection.IceServer> iceServers) {
794 if (isTerminated()) {
795 Log.w(
796 Config.LOGTAG,
797 id.account.getJid().asBareJid()
798 + ": ICE servers got discovered when session was already terminated. nothing to do.");
799 return;
800 }
801 try {
802 setupWebRTC(media, iceServers);
803 } catch (final WebRTCWrapper.InitializationException e) {
804 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
805 webRTCWrapper.close();
806 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
807 return;
808 }
809 final org.webrtc.SessionDescription sdp =
810 new org.webrtc.SessionDescription(
811 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
812 try {
813 this.webRTCWrapper.setRemoteDescription(sdp).get();
814 addIceCandidatesFromBlackLog();
815 org.webrtc.SessionDescription webRTCSessionDescription =
816 this.webRTCWrapper.setLocalDescription().get();
817 prepareSessionAccept(webRTCSessionDescription);
818 } catch (final Exception e) {
819 failureToAcceptSession(e);
820 }
821 }
822
823 private void failureToAcceptSession(final Throwable throwable) {
824 if (isTerminated()) {
825 return;
826 }
827 final Throwable rootCause = Throwables.getRootCause(throwable);
828 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
829 webRTCWrapper.close();
830 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
831 }
832
833 private void addIceCandidatesFromBlackLog() {
834 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
835 while ((foo = this.pendingIceCandidates.poll()) != null) {
836 processCandidate(foo);
837 Log.d(
838 Config.LOGTAG,
839 id.account.getJid().asBareJid() + ": added candidate from back log");
840 }
841 }
842
843 private void prepareSessionAccept(
844 final org.webrtc.SessionDescription webRTCSessionDescription) {
845 final SessionDescription sessionDescription =
846 SessionDescription.parse(webRTCSessionDescription.description);
847 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
848 this.responderRtpContentMap = respondingRtpContentMap;
849 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
850 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
851 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
852 prepareOutgoingContentMap(respondingRtpContentMap);
853 Futures.addCallback(
854 outgoingContentMapFuture,
855 new FutureCallback<RtpContentMap>() {
856 @Override
857 public void onSuccess(final RtpContentMap outgoingContentMap) {
858 sendSessionAccept(outgoingContentMap);
859 }
860
861 @Override
862 public void onFailure(@NonNull Throwable throwable) {
863 failureToAcceptSession(throwable);
864 }
865 },
866 MoreExecutors.directExecutor());
867 }
868
869 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
870 if (isTerminated()) {
871 Log.w(
872 Config.LOGTAG,
873 id.account.getJid().asBareJid()
874 + ": preparing session accept was too slow. already terminated. nothing to do.");
875 return;
876 }
877 transitionOrThrow(State.SESSION_ACCEPTED);
878 final JinglePacket sessionAccept =
879 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
880 send(sessionAccept);
881 }
882
883 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
884 final RtpContentMap rtpContentMap) {
885 if (this.omemoVerification.hasDeviceId()) {
886 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
887 verifiedPayloadFuture =
888 id.account
889 .getAxolotlService()
890 .encrypt(
891 rtpContentMap,
892 id.with,
893 omemoVerification.getDeviceId());
894 return Futures.transform(
895 verifiedPayloadFuture,
896 verifiedPayload -> {
897 omemoVerification.setOrEnsureEqual(verifiedPayload);
898 return verifiedPayload.getPayload();
899 },
900 MoreExecutors.directExecutor());
901 } else {
902 return Futures.immediateFuture(rtpContentMap);
903 }
904 }
905
906 synchronized void deliveryMessage(
907 final Jid from,
908 final Element message,
909 final String serverMessageId,
910 final long timestamp) {
911 Log.d(
912 Config.LOGTAG,
913 id.account.getJid().asBareJid()
914 + ": delivered message to JingleRtpConnection "
915 + message);
916 switch (message.getName()) {
917 case "propose":
918 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
919 break;
920 case "proceed":
921 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
922 break;
923 case "retract":
924 receiveRetract(from, serverMessageId, timestamp);
925 break;
926 case "reject":
927 receiveReject(from, serverMessageId, timestamp);
928 break;
929 case "accept":
930 receiveAccept(from, serverMessageId, timestamp);
931 break;
932 default:
933 break;
934 }
935 }
936
937 void deliverFailedProceed(final String message) {
938 Log.d(
939 Config.LOGTAG,
940 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
941 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
942 webRTCWrapper.close();
943 Log.d(
944 Config.LOGTAG,
945 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
946 this.finish();
947 }
948 }
949
950 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
951 final boolean originatedFromMyself =
952 from.asBareJid().equals(id.account.getJid().asBareJid());
953 if (originatedFromMyself) {
954 if (transition(State.ACCEPTED)) {
955 if (serverMsgId != null) {
956 this.message.setServerMsgId(serverMsgId);
957 }
958 this.message.setTime(timestamp);
959 this.message.setCarbon(true); // indicate that call was accepted on other device
960 this.writeLogMessageSuccess(0);
961 this.xmppConnectionService
962 .getNotificationService()
963 .cancelIncomingCallNotification();
964 this.finish();
965 } else {
966 Log.d(
967 Config.LOGTAG,
968 id.account.getJid().asBareJid()
969 + ": unable to transition to accept because already in state="
970 + this.state);
971 }
972 } else {
973 Log.d(
974 Config.LOGTAG,
975 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
976 }
977 }
978
979 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
980 final boolean originatedFromMyself =
981 from.asBareJid().equals(id.account.getJid().asBareJid());
982 // reject from another one of my clients
983 if (originatedFromMyself) {
984 receiveRejectFromMyself(serverMsgId, timestamp);
985 } else if (isInitiator()) {
986 if (from.equals(id.with)) {
987 receiveRejectFromResponder();
988 } else {
989 Log.d(
990 Config.LOGTAG,
991 id.account.getJid()
992 + ": ignoring reject from "
993 + from
994 + " for session with "
995 + id.with);
996 }
997 } else {
998 Log.d(
999 Config.LOGTAG,
1000 id.account.getJid()
1001 + ": ignoring reject from "
1002 + from
1003 + " for session with "
1004 + id.with);
1005 }
1006 }
1007
1008 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1009 if (transition(State.REJECTED)) {
1010 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1011 this.finish();
1012 if (serverMsgId != null) {
1013 this.message.setServerMsgId(serverMsgId);
1014 }
1015 this.message.setTime(timestamp);
1016 this.message.setCarbon(true); // indicate that call was rejected on other device
1017 writeLogMessageMissed();
1018 } else {
1019 Log.d(
1020 Config.LOGTAG,
1021 "not able to transition into REJECTED because already in " + this.state);
1022 }
1023 }
1024
1025 private void receiveRejectFromResponder() {
1026 if (isInState(State.PROCEED)) {
1027 Log.d(
1028 Config.LOGTAG,
1029 id.account.getJid()
1030 + ": received reject while still in proceed. callee reconsidered");
1031 closeTransitionLogFinish(State.REJECTED_RACED);
1032 return;
1033 }
1034 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1035 Log.d(
1036 Config.LOGTAG,
1037 id.account.getJid()
1038 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1039 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1040 return;
1041 }
1042 Log.d(
1043 Config.LOGTAG,
1044 id.account.getJid()
1045 + ": ignoring reject from responder because already in state "
1046 + this.state);
1047 }
1048
1049 private void receivePropose(
1050 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1051 final boolean originatedFromMyself =
1052 from.asBareJid().equals(id.account.getJid().asBareJid());
1053 if (originatedFromMyself) {
1054 Log.d(
1055 Config.LOGTAG,
1056 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1057 } else if (transition(
1058 State.PROPOSED,
1059 () -> {
1060 final Collection<RtpDescription> descriptions =
1061 Collections2.transform(
1062 Collections2.filter(
1063 propose.getDescriptions(),
1064 d -> d instanceof RtpDescription),
1065 input -> (RtpDescription) input);
1066 final Collection<Media> media =
1067 Collections2.transform(descriptions, RtpDescription::getMedia);
1068 Preconditions.checkState(
1069 !media.contains(Media.UNKNOWN),
1070 "RTP descriptions contain unknown media");
1071 Log.d(
1072 Config.LOGTAG,
1073 id.account.getJid().asBareJid()
1074 + ": received session proposal from "
1075 + from
1076 + " for "
1077 + media);
1078 this.proposedMedia = Sets.newHashSet(media);
1079 })) {
1080 if (serverMsgId != null) {
1081 this.message.setServerMsgId(serverMsgId);
1082 }
1083 this.message.setTime(timestamp);
1084 startRinging();
1085 } else {
1086 Log.d(
1087 Config.LOGTAG,
1088 id.account.getJid()
1089 + ": ignoring session proposal because already in "
1090 + state);
1091 }
1092 }
1093
1094 private void startRinging() {
1095 Log.d(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": received call from "
1099 + id.with
1100 + ". start ringing");
1101 ringingTimeoutFuture =
1102 jingleConnectionManager.schedule(
1103 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1104 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1105 }
1106
1107 private synchronized void ringingTimeout() {
1108 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1109 switch (this.state) {
1110 case PROPOSED:
1111 message.markUnread();
1112 rejectCallFromProposed();
1113 break;
1114 case SESSION_INITIALIZED:
1115 message.markUnread();
1116 rejectCallFromSessionInitiate();
1117 break;
1118 }
1119 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1120 }
1121
1122 private void cancelRingingTimeout() {
1123 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1124 if (future != null && !future.isCancelled()) {
1125 future.cancel(false);
1126 }
1127 }
1128
1129 private void receiveProceed(
1130 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1131 final Set<Media> media =
1132 Preconditions.checkNotNull(
1133 this.proposedMedia, "Proposed media has to be set before handling proceed");
1134 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1135 if (from.equals(id.with)) {
1136 if (isInitiator()) {
1137 if (transition(State.PROCEED)) {
1138 if (serverMsgId != null) {
1139 this.message.setServerMsgId(serverMsgId);
1140 }
1141 this.message.setTime(timestamp);
1142 final Integer remoteDeviceId = proceed.getDeviceId();
1143 if (isOmemoEnabled()) {
1144 this.omemoVerification.setDeviceId(remoteDeviceId);
1145 } else {
1146 if (remoteDeviceId != null) {
1147 Log.d(
1148 Config.LOGTAG,
1149 id.account.getJid().asBareJid()
1150 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1151 }
1152 this.omemoVerification.setDeviceId(null);
1153 }
1154 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1155 } else {
1156 Log.d(
1157 Config.LOGTAG,
1158 String.format(
1159 "%s: ignoring proceed because already in %s",
1160 id.account.getJid().asBareJid(), this.state));
1161 }
1162 } else {
1163 Log.d(
1164 Config.LOGTAG,
1165 String.format(
1166 "%s: ignoring proceed because we were not initializing",
1167 id.account.getJid().asBareJid()));
1168 }
1169 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1170 if (transition(State.ACCEPTED)) {
1171 Log.d(
1172 Config.LOGTAG,
1173 id.account.getJid().asBareJid()
1174 + ": moved session with "
1175 + id.with
1176 + " into state accepted after received carbon copied procced");
1177 this.xmppConnectionService
1178 .getNotificationService()
1179 .cancelIncomingCallNotification();
1180 this.finish();
1181 }
1182 } else {
1183 Log.d(
1184 Config.LOGTAG,
1185 String.format(
1186 "%s: ignoring proceed from %s. was expected from %s",
1187 id.account.getJid().asBareJid(), from, id.with));
1188 }
1189 }
1190
1191 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1192 if (from.equals(id.with)) {
1193 final State target =
1194 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1195 if (transition(target)) {
1196 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1197 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1198 Log.d(
1199 Config.LOGTAG,
1200 id.account.getJid().asBareJid()
1201 + ": session with "
1202 + id.with
1203 + " has been retracted (serverMsgId="
1204 + serverMsgId
1205 + ")");
1206 if (serverMsgId != null) {
1207 this.message.setServerMsgId(serverMsgId);
1208 }
1209 this.message.setTime(timestamp);
1210 if (target == State.RETRACTED) {
1211 this.message.markUnread();
1212 }
1213 writeLogMessageMissed();
1214 finish();
1215 } else {
1216 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1217 }
1218 } else {
1219 // TODO parse retract from self
1220 Log.d(
1221 Config.LOGTAG,
1222 id.account.getJid().asBareJid()
1223 + ": received retract from "
1224 + from
1225 + ". expected retract from"
1226 + id.with
1227 + ". ignoring");
1228 }
1229 }
1230
1231 public void sendSessionInitiate() {
1232 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1233 }
1234
1235 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1236 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1237 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1238 }
1239
1240 private synchronized void sendSessionInitiate(
1241 final Set<Media> media,
1242 final State targetState,
1243 final List<PeerConnection.IceServer> iceServers) {
1244 if (isTerminated()) {
1245 Log.w(
1246 Config.LOGTAG,
1247 id.account.getJid().asBareJid()
1248 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1249 return;
1250 }
1251 try {
1252 setupWebRTC(media, iceServers);
1253 } catch (final WebRTCWrapper.InitializationException e) {
1254 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1255 webRTCWrapper.close();
1256 sendRetract(Reason.ofThrowable(e));
1257 return;
1258 }
1259 try {
1260 org.webrtc.SessionDescription webRTCSessionDescription =
1261 this.webRTCWrapper.setLocalDescription().get();
1262 prepareSessionInitiate(webRTCSessionDescription, targetState);
1263 } catch (final Exception e) {
1264 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1265 // exceptions
1266 failureToInitiateSession(e, targetState);
1267 }
1268 }
1269
1270 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1271 if (isTerminated()) {
1272 return;
1273 }
1274 Log.d(
1275 Config.LOGTAG,
1276 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1277 Throwables.getRootCause(throwable));
1278 webRTCWrapper.close();
1279 final Reason reason = Reason.ofThrowable(throwable);
1280 if (isInState(targetState)) {
1281 sendSessionTerminate(reason, throwable.getMessage());
1282 } else {
1283 sendRetract(reason);
1284 }
1285 }
1286
1287 private void sendRetract(final Reason reason) {
1288 // TODO embed reason into retract
1289 sendJingleMessage("retract", id.with.asBareJid());
1290 transitionOrThrow(reasonToState(reason));
1291 this.finish();
1292 }
1293
1294 private void prepareSessionInitiate(
1295 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1296 final SessionDescription sessionDescription =
1297 SessionDescription.parse(webRTCSessionDescription.description);
1298 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1299 this.initiatorRtpContentMap = rtpContentMap;
1300 //TODO delay ready to receive ice until after session-init
1301 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1302 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1303 encryptSessionInitiate(rtpContentMap);
1304 Futures.addCallback(
1305 outgoingContentMapFuture,
1306 new FutureCallback<RtpContentMap>() {
1307 @Override
1308 public void onSuccess(final RtpContentMap outgoingContentMap) {
1309 sendSessionInitiate(outgoingContentMap, targetState);
1310 }
1311
1312 @Override
1313 public void onFailure(@NonNull final Throwable throwable) {
1314 failureToInitiateSession(throwable, targetState);
1315 }
1316 },
1317 MoreExecutors.directExecutor());
1318 }
1319
1320 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1321 if (isTerminated()) {
1322 Log.w(
1323 Config.LOGTAG,
1324 id.account.getJid().asBareJid()
1325 + ": preparing session was too slow. already terminated. nothing to do.");
1326 return;
1327 }
1328 this.transitionOrThrow(targetState);
1329 final JinglePacket sessionInitiate =
1330 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1331 send(sessionInitiate);
1332 }
1333
1334 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1335 final RtpContentMap rtpContentMap) {
1336 if (this.omemoVerification.hasDeviceId()) {
1337 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1338 verifiedPayloadFuture =
1339 id.account
1340 .getAxolotlService()
1341 .encrypt(
1342 rtpContentMap,
1343 id.with,
1344 omemoVerification.getDeviceId());
1345 final ListenableFuture<RtpContentMap> future =
1346 Futures.transform(
1347 verifiedPayloadFuture,
1348 verifiedPayload -> {
1349 omemoVerification.setSessionFingerprint(
1350 verifiedPayload.getFingerprint());
1351 return verifiedPayload.getPayload();
1352 },
1353 MoreExecutors.directExecutor());
1354 if (Config.REQUIRE_RTP_VERIFICATION) {
1355 return future;
1356 }
1357 return Futures.catching(
1358 future,
1359 CryptoFailedException.class,
1360 e -> {
1361 Log.w(
1362 Config.LOGTAG,
1363 id.account.getJid().asBareJid()
1364 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1365 e);
1366 return rtpContentMap;
1367 },
1368 MoreExecutors.directExecutor());
1369 } else {
1370 return Futures.immediateFuture(rtpContentMap);
1371 }
1372 }
1373
1374 private void sendSessionTerminate(final Reason reason) {
1375 sendSessionTerminate(reason, null);
1376 }
1377
1378 private void sendSessionTerminate(final Reason reason, final String text) {
1379 final State previous = this.state;
1380 final State target = reasonToState(reason);
1381 transitionOrThrow(target);
1382 if (previous != State.NULL) {
1383 writeLogMessage(target);
1384 }
1385 final JinglePacket jinglePacket =
1386 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1387 jinglePacket.setReason(reason, text);
1388 Log.d(Config.LOGTAG, jinglePacket.toString());
1389 send(jinglePacket);
1390 finish();
1391 }
1392
1393 private void sendTransportInfo(
1394 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1395 final RtpContentMap transportInfo;
1396 try {
1397 final RtpContentMap rtpContentMap =
1398 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1399 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1400 } catch (final Exception e) {
1401 Log.d(
1402 Config.LOGTAG,
1403 id.account.getJid().asBareJid()
1404 + ": unable to prepare transport-info from candidate for content="
1405 + contentName);
1406 return;
1407 }
1408 final JinglePacket jinglePacket =
1409 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1410 send(jinglePacket);
1411 }
1412
1413 private void send(final JinglePacket jinglePacket) {
1414 jinglePacket.setTo(id.with);
1415 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1416 }
1417
1418 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1419 if (response.getType() == IqPacket.TYPE.ERROR) {
1420 handleIqErrorResponse(response);
1421 return;
1422 }
1423 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1424 handleIqTimeoutResponse(response);
1425 }
1426 }
1427
1428 private void handleIqErrorResponse(final IqPacket response) {
1429 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1430 final String errorCondition = response.getErrorCondition();
1431 Log.d(
1432 Config.LOGTAG,
1433 id.account.getJid().asBareJid()
1434 + ": received IQ-error from "
1435 + response.getFrom()
1436 + " in RTP session. "
1437 + errorCondition);
1438 if (isTerminated()) {
1439 Log.i(
1440 Config.LOGTAG,
1441 id.account.getJid().asBareJid()
1442 + ": ignoring error because session was already terminated");
1443 return;
1444 }
1445 this.webRTCWrapper.close();
1446 final State target;
1447 if (Arrays.asList(
1448 "service-unavailable",
1449 "recipient-unavailable",
1450 "remote-server-not-found",
1451 "remote-server-timeout")
1452 .contains(errorCondition)) {
1453 target = State.TERMINATED_CONNECTIVITY_ERROR;
1454 } else {
1455 target = State.TERMINATED_APPLICATION_FAILURE;
1456 }
1457 transitionOrThrow(target);
1458 this.finish();
1459 }
1460
1461 private void handleIqTimeoutResponse(final IqPacket response) {
1462 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1463 Log.d(
1464 Config.LOGTAG,
1465 id.account.getJid().asBareJid()
1466 + ": received IQ timeout in RTP session with "
1467 + id.with
1468 + ". terminating with connectivity error");
1469 if (isTerminated()) {
1470 Log.i(
1471 Config.LOGTAG,
1472 id.account.getJid().asBareJid()
1473 + ": ignoring error because session was already terminated");
1474 return;
1475 }
1476 this.webRTCWrapper.close();
1477 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1478 this.finish();
1479 }
1480
1481 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1482 Log.d(
1483 Config.LOGTAG,
1484 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1485 this.webRTCWrapper.close();
1486 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1487 respondWithOutOfOrder(jinglePacket);
1488 this.finish();
1489 }
1490
1491 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1492 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1493 }
1494
1495 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1496 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1497 }
1498
1499 void respondWithJingleError(
1500 final IqPacket original,
1501 String jingleCondition,
1502 String condition,
1503 String conditionType) {
1504 jingleConnectionManager.respondWithJingleError(
1505 id.account, original, jingleCondition, condition, conditionType);
1506 }
1507
1508 private void respondOk(final JinglePacket jinglePacket) {
1509 xmppConnectionService.sendIqPacket(
1510 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1511 }
1512
1513 public RtpEndUserState getEndUserState() {
1514 switch (this.state) {
1515 case NULL:
1516 case PROPOSED:
1517 case SESSION_INITIALIZED:
1518 if (isInitiator()) {
1519 return RtpEndUserState.RINGING;
1520 } else {
1521 return RtpEndUserState.INCOMING_CALL;
1522 }
1523 case PROCEED:
1524 if (isInitiator()) {
1525 return RtpEndUserState.RINGING;
1526 } else {
1527 return RtpEndUserState.ACCEPTING_CALL;
1528 }
1529 case SESSION_INITIALIZED_PRE_APPROVED:
1530 if (isInitiator()) {
1531 return RtpEndUserState.RINGING;
1532 } else {
1533 return RtpEndUserState.CONNECTING;
1534 }
1535 case SESSION_ACCEPTED:
1536 return getPeerConnectionStateAsEndUserState();
1537 case REJECTED:
1538 case REJECTED_RACED:
1539 case TERMINATED_DECLINED_OR_BUSY:
1540 if (isInitiator()) {
1541 return RtpEndUserState.DECLINED_OR_BUSY;
1542 } else {
1543 return RtpEndUserState.ENDED;
1544 }
1545 case TERMINATED_SUCCESS:
1546 case ACCEPTED:
1547 case RETRACTED:
1548 case TERMINATED_CANCEL_OR_TIMEOUT:
1549 return RtpEndUserState.ENDED;
1550 case RETRACTED_RACED:
1551 if (isInitiator()) {
1552 return RtpEndUserState.ENDED;
1553 } else {
1554 return RtpEndUserState.RETRACTED;
1555 }
1556 case TERMINATED_CONNECTIVITY_ERROR:
1557 return zeroDuration()
1558 ? RtpEndUserState.CONNECTIVITY_ERROR
1559 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1560 case TERMINATED_APPLICATION_FAILURE:
1561 return RtpEndUserState.APPLICATION_ERROR;
1562 case TERMINATED_SECURITY_ERROR:
1563 return RtpEndUserState.SECURITY_ERROR;
1564 }
1565 throw new IllegalStateException(
1566 String.format("%s has no equivalent EndUserState", this.state));
1567 }
1568
1569 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1570 final PeerConnection.PeerConnectionState state;
1571 try {
1572 state = webRTCWrapper.getState();
1573 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1574 // We usually close the WebRTCWrapper *before* transitioning so we might still
1575 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
1576 return RtpEndUserState.ENDING_CALL;
1577 }
1578 switch (state) {
1579 case CONNECTED:
1580 return RtpEndUserState.CONNECTED;
1581 case NEW:
1582 case CONNECTING:
1583 return RtpEndUserState.CONNECTING;
1584 case CLOSED:
1585 return RtpEndUserState.ENDING_CALL;
1586 default:
1587 return zeroDuration()
1588 ? RtpEndUserState.CONNECTIVITY_ERROR
1589 : RtpEndUserState.RECONNECTING;
1590 }
1591 }
1592
1593 public Set<Media> getMedia() {
1594 final State current = getState();
1595 if (current == State.NULL) {
1596 if (isInitiator()) {
1597 return Preconditions.checkNotNull(
1598 this.proposedMedia, "RTP connection has not been initialized properly");
1599 }
1600 throw new IllegalStateException("RTP connection has not been initialized yet");
1601 }
1602 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1603 return Preconditions.checkNotNull(
1604 this.proposedMedia, "RTP connection has not been initialized properly");
1605 }
1606 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1607 if (initiatorContentMap != null) {
1608 return initiatorContentMap.getMedia();
1609 } else if (isTerminated()) {
1610 return Collections.emptySet(); // we might fail before we ever got a chance to set media
1611 } else {
1612 return Preconditions.checkNotNull(
1613 this.proposedMedia, "RTP connection has not been initialized properly");
1614 }
1615 }
1616
1617 public boolean isVerified() {
1618 final String fingerprint = this.omemoVerification.getFingerprint();
1619 if (fingerprint == null) {
1620 return false;
1621 }
1622 final FingerprintStatus status =
1623 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1624 return status != null && status.isVerified();
1625 }
1626
1627 public synchronized void acceptCall() {
1628 switch (this.state) {
1629 case PROPOSED:
1630 cancelRingingTimeout();
1631 acceptCallFromProposed();
1632 break;
1633 case SESSION_INITIALIZED:
1634 cancelRingingTimeout();
1635 acceptCallFromSessionInitialized();
1636 break;
1637 case ACCEPTED:
1638 Log.w(
1639 Config.LOGTAG,
1640 id.account.getJid().asBareJid()
1641 + ": the call has already been accepted with another client. UI was just lagging behind");
1642 break;
1643 case PROCEED:
1644 case SESSION_ACCEPTED:
1645 Log.w(
1646 Config.LOGTAG,
1647 id.account.getJid().asBareJid()
1648 + ": the call has already been accepted. user probably double tapped the UI");
1649 break;
1650 default:
1651 throw new IllegalStateException("Can not accept call from " + this.state);
1652 }
1653 }
1654
1655 public void notifyPhoneCall() {
1656 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1657 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1658 rejectCall();
1659 } else {
1660 endCall();
1661 }
1662 }
1663
1664 public synchronized void rejectCall() {
1665 if (isTerminated()) {
1666 Log.w(
1667 Config.LOGTAG,
1668 id.account.getJid().asBareJid()
1669 + ": received rejectCall() when session has already been terminated. nothing to do");
1670 return;
1671 }
1672 switch (this.state) {
1673 case PROPOSED:
1674 rejectCallFromProposed();
1675 break;
1676 case SESSION_INITIALIZED:
1677 rejectCallFromSessionInitiate();
1678 break;
1679 default:
1680 throw new IllegalStateException("Can not reject call from " + this.state);
1681 }
1682 }
1683
1684 public synchronized void endCall() {
1685 if (isTerminated()) {
1686 Log.w(
1687 Config.LOGTAG,
1688 id.account.getJid().asBareJid()
1689 + ": received endCall() when session has already been terminated. nothing to do");
1690 return;
1691 }
1692 if (isInState(State.PROPOSED) && !isInitiator()) {
1693 rejectCallFromProposed();
1694 return;
1695 }
1696 if (isInState(State.PROCEED)) {
1697 if (isInitiator()) {
1698 retractFromProceed();
1699 } else {
1700 rejectCallFromProceed();
1701 }
1702 return;
1703 }
1704 if (isInitiator()
1705 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1706 this.webRTCWrapper.close();
1707 sendSessionTerminate(Reason.CANCEL);
1708 return;
1709 }
1710 if (isInState(State.SESSION_INITIALIZED)) {
1711 rejectCallFromSessionInitiate();
1712 return;
1713 }
1714 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1715 this.webRTCWrapper.close();
1716 sendSessionTerminate(Reason.SUCCESS);
1717 return;
1718 }
1719 if (isInState(
1720 State.TERMINATED_APPLICATION_FAILURE,
1721 State.TERMINATED_CONNECTIVITY_ERROR,
1722 State.TERMINATED_DECLINED_OR_BUSY)) {
1723 Log.d(
1724 Config.LOGTAG,
1725 "ignoring request to end call because already in state " + this.state);
1726 return;
1727 }
1728 throw new IllegalStateException(
1729 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1730 }
1731
1732 private void retractFromProceed() {
1733 Log.d(Config.LOGTAG, "retract from proceed");
1734 this.sendJingleMessage("retract");
1735 closeTransitionLogFinish(State.RETRACTED_RACED);
1736 }
1737
1738 private void closeTransitionLogFinish(final State state) {
1739 this.webRTCWrapper.close();
1740 transitionOrThrow(state);
1741 writeLogMessage(state);
1742 finish();
1743 }
1744
1745 private void setupWebRTC(
1746 final Set<Media> media, final List<PeerConnection.IceServer> iceServers)
1747 throws WebRTCWrapper.InitializationException {
1748 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1749 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1750 if (media.contains(Media.VIDEO)) {
1751 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1752 } else {
1753 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1754 }
1755 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1756 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1757 }
1758
1759 private void acceptCallFromProposed() {
1760 transitionOrThrow(State.PROCEED);
1761 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1762 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1763 this.sendJingleMessage("proceed");
1764 }
1765
1766 private void rejectCallFromProposed() {
1767 transitionOrThrow(State.REJECTED);
1768 writeLogMessageMissed();
1769 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1770 this.sendJingleMessage("reject");
1771 finish();
1772 }
1773
1774 private void rejectCallFromProceed() {
1775 this.sendJingleMessage("reject");
1776 closeTransitionLogFinish(State.REJECTED_RACED);
1777 }
1778
1779 private void rejectCallFromSessionInitiate() {
1780 webRTCWrapper.close();
1781 sendSessionTerminate(Reason.DECLINE);
1782 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1783 }
1784
1785 private void sendJingleMessage(final String action) {
1786 sendJingleMessage(action, id.with);
1787 }
1788
1789 private void sendJingleMessage(final String action, final Jid to) {
1790 final MessagePacket messagePacket = new MessagePacket();
1791 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
1792 messagePacket.setTo(to);
1793 final Element intent =
1794 messagePacket
1795 .addChild(action, Namespace.JINGLE_MESSAGE)
1796 .setAttribute("id", id.sessionId);
1797 if ("proceed".equals(action)) {
1798 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1799 if (isOmemoEnabled()) {
1800 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1801 final Element device =
1802 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1803 device.setAttribute("id", deviceId);
1804 }
1805 }
1806 messagePacket.addChild("store", "urn:xmpp:hints");
1807 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1808 }
1809
1810 private boolean isOmemoEnabled() {
1811 final Conversational conversational = message.getConversation();
1812 if (conversational instanceof Conversation) {
1813 return ((Conversation) conversational).getNextEncryption()
1814 == Message.ENCRYPTION_AXOLOTL;
1815 }
1816 return false;
1817 }
1818
1819 private void acceptCallFromSessionInitialized() {
1820 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1821 sendSessionAccept();
1822 }
1823
1824 private synchronized boolean isInState(State... state) {
1825 return Arrays.asList(state).contains(this.state);
1826 }
1827
1828 private boolean transition(final State target) {
1829 return transition(target, null);
1830 }
1831
1832 private synchronized boolean transition(final State target, final Runnable runnable) {
1833 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1834 if (validTransitions != null && validTransitions.contains(target)) {
1835 this.state = target;
1836 if (runnable != null) {
1837 runnable.run();
1838 }
1839 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1840 updateEndUserState();
1841 updateOngoingCallNotification();
1842 return true;
1843 } else {
1844 return false;
1845 }
1846 }
1847
1848 void transitionOrThrow(final State target) {
1849 if (!transition(target)) {
1850 throw new IllegalStateException(
1851 String.format("Unable to transition from %s to %s", this.state, target));
1852 }
1853 }
1854
1855 @Override
1856 public void onIceCandidate(final IceCandidate iceCandidate) {
1857 final RtpContentMap rtpContentMap =
1858 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1859 final IceUdpTransportInfo.Credentials credentials;
1860 try {
1861 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
1862 } catch (final IllegalArgumentException e) {
1863 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
1864 return;
1865 }
1866 final String uFrag = credentials.ufrag;
1867 final IceUdpTransportInfo.Candidate candidate =
1868 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
1869 if (candidate == null) {
1870 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
1871 return;
1872 }
1873 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
1874 sendTransportInfo(iceCandidate.sdpMid, candidate);
1875 }
1876
1877 @Override
1878 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1879 Log.d(
1880 Config.LOGTAG,
1881 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1882 this.stateHistory.add(newState);
1883 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1884 this.sessionDuration.start();
1885 updateOngoingCallNotification();
1886 } else if (this.sessionDuration.isRunning()) {
1887 this.sessionDuration.stop();
1888 updateOngoingCallNotification();
1889 }
1890
1891 final boolean neverConnected =
1892 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1893
1894 if (newState == PeerConnection.PeerConnectionState.FAILED) {
1895 if (neverConnected) {
1896 if (isTerminated()) {
1897 Log.d(
1898 Config.LOGTAG,
1899 id.account.getJid().asBareJid()
1900 + ": not sending session-terminate after connectivity error because session is already in state "
1901 + this.state);
1902 return;
1903 }
1904 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1905 return;
1906 } else {
1907 webRTCWrapper.restartIce();
1908 }
1909 }
1910 updateEndUserState();
1911 }
1912
1913 @Override
1914 public void onRenegotiationNeeded() {
1915 this.webRTCWrapper.execute(this::renegotiate);
1916 }
1917
1918 private void renegotiate() {
1919 //TODO needs to be called only for ice restarts; maybe in the call to restartICe()
1920 this.stateHistory.clear();
1921 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1922 final SessionDescription sessionDescription;
1923 try {
1924 sessionDescription = setLocalSessionDescription();
1925 } catch (final Exception e) {
1926 final Throwable cause = Throwables.getRootCause(e);
1927 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
1928 webRTCWrapper.close();
1929 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
1930 return;
1931 }
1932 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
1933 final RtpContentMap currentContentMap = getLocalContentMap();
1934 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
1935 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
1936
1937 Log.d(
1938 Config.LOGTAG,
1939 id.getAccount().getJid().asBareJid()
1940 + ": renegotiate. iceRestart="
1941 + iceRestart
1942 + " content id diff="
1943 + diff);
1944
1945 if (diff.hasModifications() && iceRestart) {
1946 webRTCWrapper.close();
1947 sendSessionTerminate(Reason.FAILED_APPLICATION, "WebRTC unexpectedly tried to modify content and transport at once");
1948 return;
1949 }
1950
1951 if (iceRestart) {
1952 initiateIceRestart(rtpContentMap);
1953 return;
1954 }
1955
1956 if (diff.added.size() > 0) {
1957 sendContentAdd(rtpContentMap);
1958 }
1959
1960 }
1961
1962 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
1963 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
1964 final JinglePacket jinglePacket =
1965 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1966 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
1967 jinglePacket.setTo(id.with);
1968 xmppConnectionService.sendIqPacket(
1969 id.account,
1970 jinglePacket,
1971 (account, response) -> {
1972 if (response.getType() == IqPacket.TYPE.RESULT) {
1973 Log.d(Config.LOGTAG, "received success to our ice restart");
1974 setLocalContentMap(rtpContentMap);
1975 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1976 return;
1977 }
1978 if (response.getType() == IqPacket.TYPE.ERROR) {
1979 final Element error = response.findChild("error");
1980 if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
1981 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
1982 return;
1983 }
1984 handleIqErrorResponse(response);
1985 }
1986 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1987 handleIqTimeoutResponse(response);
1988 }
1989 });
1990 }
1991
1992 private void sendContentAdd(final RtpContentMap rtpContentMap) {
1993
1994 }
1995
1996 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
1997 if (isInitiator()) {
1998 this.initiatorRtpContentMap = rtpContentMap;
1999 } else {
2000 this.responderRtpContentMap = rtpContentMap;
2001 }
2002 }
2003
2004 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2005 if (isInitiator()) {
2006 this.responderRtpContentMap = rtpContentMap;
2007 } else {
2008 this.initiatorRtpContentMap = rtpContentMap;
2009 }
2010 }
2011
2012 private SessionDescription setLocalSessionDescription()
2013 throws ExecutionException, InterruptedException {
2014 final org.webrtc.SessionDescription sessionDescription =
2015 this.webRTCWrapper.setLocalDescription().get();
2016 return SessionDescription.parse(sessionDescription.description);
2017 }
2018
2019 private void closeWebRTCSessionAfterFailedConnection() {
2020 this.webRTCWrapper.close();
2021 synchronized (this) {
2022 if (isTerminated()) {
2023 Log.d(
2024 Config.LOGTAG,
2025 id.account.getJid().asBareJid()
2026 + ": no need to send session-terminate after failed connection. Other party already did");
2027 return;
2028 }
2029 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2030 }
2031 }
2032
2033 public boolean zeroDuration() {
2034 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2035 }
2036
2037 public long getCallDuration() {
2038 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2039 }
2040
2041 public AppRTCAudioManager getAudioManager() {
2042 return webRTCWrapper.getAudioManager();
2043 }
2044
2045 public boolean isMicrophoneEnabled() {
2046 return webRTCWrapper.isMicrophoneEnabled();
2047 }
2048
2049 public boolean setMicrophoneEnabled(final boolean enabled) {
2050 return webRTCWrapper.setMicrophoneEnabled(enabled);
2051 }
2052
2053 public boolean isVideoEnabled() {
2054 return webRTCWrapper.isVideoEnabled();
2055 }
2056
2057 public void setVideoEnabled(final boolean enabled) {
2058 webRTCWrapper.setVideoEnabled(enabled);
2059 }
2060
2061 public boolean isCameraSwitchable() {
2062 return webRTCWrapper.isCameraSwitchable();
2063 }
2064
2065 public boolean isFrontCamera() {
2066 return webRTCWrapper.isFrontCamera();
2067 }
2068
2069 public ListenableFuture<Boolean> switchCamera() {
2070 return webRTCWrapper.switchCamera();
2071 }
2072
2073 @Override
2074 public void onAudioDeviceChanged(
2075 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2076 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2077 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2078 selectedAudioDevice, availableAudioDevices);
2079 }
2080
2081 private void updateEndUserState() {
2082 final RtpEndUserState endUserState = getEndUserState();
2083 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2084 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2085 id.account, id.with, id.sessionId, endUserState);
2086 }
2087
2088 private void updateOngoingCallNotification() {
2089 final State state = this.state;
2090 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2091 final boolean reconnecting;
2092 if (state == State.SESSION_ACCEPTED) {
2093 reconnecting =
2094 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2095 } else {
2096 reconnecting = false;
2097 }
2098 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2099 } else {
2100 xmppConnectionService.removeOngoingCall();
2101 }
2102 }
2103
2104 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2105 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2106 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2107 request.setTo(id.account.getDomain());
2108 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2109 xmppConnectionService.sendIqPacket(
2110 id.account,
2111 request,
2112 (account, response) -> {
2113 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2114 new ImmutableList.Builder<>();
2115 if (response.getType() == IqPacket.TYPE.RESULT) {
2116 final Element services =
2117 response.findChild(
2118 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2119 final List<Element> children =
2120 services == null
2121 ? Collections.emptyList()
2122 : services.getChildren();
2123 for (final Element child : children) {
2124 if ("service".equals(child.getName())) {
2125 final String type = child.getAttribute("type");
2126 final String host = child.getAttribute("host");
2127 final String sport = child.getAttribute("port");
2128 final Integer port =
2129 sport == null ? null : Ints.tryParse(sport);
2130 final String transport = child.getAttribute("transport");
2131 final String username = child.getAttribute("username");
2132 final String password = child.getAttribute("password");
2133 if (Strings.isNullOrEmpty(host) || port == null) {
2134 continue;
2135 }
2136 if (port < 0 || port > 65535) {
2137 continue;
2138 }
2139 if (Arrays.asList("stun", "stuns", "turn", "turns")
2140 .contains(type)
2141 && Arrays.asList("udp", "tcp").contains(transport)) {
2142 if (Arrays.asList("stuns", "turns").contains(type)
2143 && "udp".equals(transport)) {
2144 Log.d(
2145 Config.LOGTAG,
2146 id.account.getJid().asBareJid()
2147 + ": skipping invalid combination of udp/tls in external services");
2148 continue;
2149 }
2150 final PeerConnection.IceServer.Builder iceServerBuilder =
2151 PeerConnection.IceServer.builder(
2152 String.format(
2153 "%s:%s:%s?transport=%s",
2154 type,
2155 IP.wrapIPv6(host),
2156 port,
2157 transport));
2158 iceServerBuilder.setTlsCertPolicy(
2159 PeerConnection.TlsCertPolicy
2160 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2161 if (username != null && password != null) {
2162 iceServerBuilder.setUsername(username);
2163 iceServerBuilder.setPassword(password);
2164 } else if (Arrays.asList("turn", "turns").contains(type)) {
2165 // The WebRTC spec requires throwing an
2166 // InvalidAccessError when username (from libwebrtc
2167 // source coder)
2168 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2169 Log.d(
2170 Config.LOGTAG,
2171 id.account.getJid().asBareJid()
2172 + ": skipping "
2173 + type
2174 + "/"
2175 + transport
2176 + " without username and password");
2177 continue;
2178 }
2179 final PeerConnection.IceServer iceServer =
2180 iceServerBuilder.createIceServer();
2181 Log.d(
2182 Config.LOGTAG,
2183 id.account.getJid().asBareJid()
2184 + ": discovered ICE Server: "
2185 + iceServer);
2186 listBuilder.add(iceServer);
2187 }
2188 }
2189 }
2190 }
2191 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2192 if (iceServers.size() == 0) {
2193 Log.w(
2194 Config.LOGTAG,
2195 id.account.getJid().asBareJid()
2196 + ": no ICE server found "
2197 + response);
2198 }
2199 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2200 });
2201 } else {
2202 Log.w(
2203 Config.LOGTAG,
2204 id.account.getJid().asBareJid() + ": has no external service discovery");
2205 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2206 }
2207 }
2208
2209 private void finish() {
2210 if (isTerminated()) {
2211 this.cancelRingingTimeout();
2212 this.webRTCWrapper.verifyClosed();
2213 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2214 this.jingleConnectionManager.finishConnectionOrThrow(this);
2215 } else {
2216 throw new IllegalStateException(
2217 String.format("Unable to call finish from %s", this.state));
2218 }
2219 }
2220
2221 private void writeLogMessage(final State state) {
2222 final long duration = getCallDuration();
2223 if (state == State.TERMINATED_SUCCESS
2224 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2225 writeLogMessageSuccess(duration);
2226 } else {
2227 writeLogMessageMissed();
2228 }
2229 }
2230
2231 private void writeLogMessageSuccess(final long duration) {
2232 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2233 this.writeMessage();
2234 }
2235
2236 private void writeLogMessageMissed() {
2237 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2238 this.writeMessage();
2239 }
2240
2241 private void writeMessage() {
2242 final Conversational conversational = message.getConversation();
2243 if (conversational instanceof Conversation) {
2244 ((Conversation) conversational).add(this.message);
2245 xmppConnectionService.createMessageAsync(message);
2246 xmppConnectionService.updateConversationUi();
2247 } else {
2248 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2249 }
2250 }
2251
2252 public State getState() {
2253 return this.state;
2254 }
2255
2256 boolean isTerminated() {
2257 return TERMINATED.contains(this.state);
2258 }
2259
2260 public Optional<VideoTrack> getLocalVideoTrack() {
2261 return webRTCWrapper.getLocalVideoTrack();
2262 }
2263
2264 public Optional<VideoTrack> getRemoteVideoTrack() {
2265 return webRTCWrapper.getRemoteVideoTrack();
2266 }
2267
2268 public EglBase.Context getEglBaseContext() {
2269 return webRTCWrapper.getEglBaseContext();
2270 }
2271
2272 void setProposedMedia(final Set<Media> media) {
2273 this.proposedMedia = media;
2274 }
2275
2276 public void fireStateUpdate() {
2277 final RtpEndUserState endUserState = getEndUserState();
2278 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2279 id.account, id.with, id.sessionId, endUserState);
2280 }
2281
2282 private interface OnIceServersDiscovered {
2283 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2284 }
2285}