1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Multimap;
22import com.google.common.collect.Sets;
23import com.google.common.primitives.Ints;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.PeerConnection;
32import org.webrtc.VideoTrack;
33
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
66import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
67import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
68import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
71import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
72import eu.siacs.conversations.xmpp.stanzas.IqPacket;
73import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
74
75public class JingleRtpConnection extends AbstractJingleConnection
76 implements WebRTCWrapper.EventCallback {
77
78 public static final List<State> STATES_SHOWING_ONGOING_CALL =
79 Arrays.asList(
80 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82 private static final List<State> TERMINATED =
83 Arrays.asList(
84 State.ACCEPTED,
85 State.REJECTED,
86 State.REJECTED_RACED,
87 State.RETRACTED,
88 State.RETRACTED_RACED,
89 State.TERMINATED_SUCCESS,
90 State.TERMINATED_DECLINED_OR_BUSY,
91 State.TERMINATED_CONNECTIVITY_ERROR,
92 State.TERMINATED_CANCEL_OR_TIMEOUT,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR);
95
96 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
97
98 static {
99 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
100 new ImmutableMap.Builder<>();
101 transitionBuilder.put(
102 State.NULL,
103 ImmutableList.of(
104 State.PROPOSED,
105 State.SESSION_INITIALIZED,
106 State.TERMINATED_APPLICATION_FAILURE,
107 State.TERMINATED_SECURITY_ERROR));
108 transitionBuilder.put(
109 State.PROPOSED,
110 ImmutableList.of(
111 State.ACCEPTED,
112 State.PROCEED,
113 State.REJECTED,
114 State.RETRACTED,
115 State.TERMINATED_APPLICATION_FAILURE,
116 State.TERMINATED_SECURITY_ERROR,
117 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
118 // rebinds
119 ));
120 transitionBuilder.put(
121 State.PROCEED,
122 ImmutableList.of(
123 State.REJECTED_RACED,
124 State.RETRACTED_RACED,
125 State.SESSION_INITIALIZED_PRE_APPROVED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_APPLICATION_FAILURE,
128 State.TERMINATED_SECURITY_ERROR,
129 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
130 // bounces of the proceed message
131 ));
132 transitionBuilder.put(
133 State.SESSION_INITIALIZED,
134 ImmutableList.of(
135 State.SESSION_ACCEPTED,
136 State.TERMINATED_SUCCESS,
137 State.TERMINATED_DECLINED_OR_BUSY,
138 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
139 // and IQ timeouts
140 State.TERMINATED_CANCEL_OR_TIMEOUT,
141 State.TERMINATED_APPLICATION_FAILURE,
142 State.TERMINATED_SECURITY_ERROR));
143 transitionBuilder.put(
144 State.SESSION_INITIALIZED_PRE_APPROVED,
145 ImmutableList.of(
146 State.SESSION_ACCEPTED,
147 State.TERMINATED_SUCCESS,
148 State.TERMINATED_DECLINED_OR_BUSY,
149 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
150 // and IQ timeouts
151 State.TERMINATED_CANCEL_OR_TIMEOUT,
152 State.TERMINATED_APPLICATION_FAILURE,
153 State.TERMINATED_SECURITY_ERROR));
154 transitionBuilder.put(
155 State.SESSION_ACCEPTED,
156 ImmutableList.of(
157 State.TERMINATED_SUCCESS,
158 State.TERMINATED_DECLINED_OR_BUSY,
159 State.TERMINATED_CONNECTIVITY_ERROR,
160 State.TERMINATED_CANCEL_OR_TIMEOUT,
161 State.TERMINATED_APPLICATION_FAILURE,
162 State.TERMINATED_SECURITY_ERROR));
163 VALID_TRANSITIONS = transitionBuilder.build();
164 }
165
166 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
167 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
168 pendingIceCandidates = new LinkedList<>();
169 private final OmemoVerification omemoVerification = new OmemoVerification();
170 private final Message message;
171 private State state = State.NULL;
172 private Set<Media> proposedMedia;
173 private RtpContentMap initiatorRtpContentMap;
174 private RtpContentMap responderRtpContentMap;
175 private RtpContentMap incomingContentAdd;
176 private RtpContentMap outgoingContentAdd;
177 private IceUdpTransportInfo.Setup peerDtlsSetup;
178 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
179 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
180 private ScheduledFuture<?> ringingTimeoutFuture;
181
182 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
183 super(jingleConnectionManager, id, initiator);
184 final Conversation conversation =
185 jingleConnectionManager
186 .getXmppConnectionService()
187 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
188 this.message =
189 new Message(
190 conversation,
191 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
192 Message.TYPE_RTP_SESSION,
193 id.sessionId);
194 }
195
196 private static State reasonToState(Reason reason) {
197 switch (reason) {
198 case SUCCESS:
199 return State.TERMINATED_SUCCESS;
200 case DECLINE:
201 case BUSY:
202 return State.TERMINATED_DECLINED_OR_BUSY;
203 case CANCEL:
204 case TIMEOUT:
205 return State.TERMINATED_CANCEL_OR_TIMEOUT;
206 case SECURITY_ERROR:
207 return State.TERMINATED_SECURITY_ERROR;
208 case FAILED_APPLICATION:
209 case UNSUPPORTED_TRANSPORTS:
210 case UNSUPPORTED_APPLICATIONS:
211 return State.TERMINATED_APPLICATION_FAILURE;
212 default:
213 return State.TERMINATED_CONNECTIVITY_ERROR;
214 }
215 }
216
217 @Override
218 synchronized void deliverPacket(final JinglePacket jinglePacket) {
219 switch (jinglePacket.getAction()) {
220 case SESSION_INITIATE:
221 receiveSessionInitiate(jinglePacket);
222 break;
223 case TRANSPORT_INFO:
224 receiveTransportInfo(jinglePacket);
225 break;
226 case SESSION_ACCEPT:
227 receiveSessionAccept(jinglePacket);
228 break;
229 case SESSION_TERMINATE:
230 receiveSessionTerminate(jinglePacket);
231 break;
232 case CONTENT_ADD:
233 receiveContentAdd(jinglePacket);
234 break;
235 case CONTENT_ACCEPT:
236 receiveContentAccept(jinglePacket);
237 break;
238 case CONTENT_REJECT:
239 receiveContentReject(jinglePacket);
240 break;
241 case CONTENT_REMOVE:
242 receiveContentRemove(jinglePacket);
243 break;
244 case CONTENT_MODIFY:
245 receiveContentModify(jinglePacket);
246 break;
247 default:
248 respondOk(jinglePacket);
249 Log.d(
250 Config.LOGTAG,
251 String.format(
252 "%s: received unhandled jingle action %s",
253 id.account.getJid().asBareJid(), jinglePacket.getAction()));
254 break;
255 }
256 }
257
258 @Override
259 synchronized void notifyRebound() {
260 if (isTerminated()) {
261 return;
262 }
263 webRTCWrapper.close();
264 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
265 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
266 }
267 if (isInState(
268 State.SESSION_INITIALIZED,
269 State.SESSION_INITIALIZED_PRE_APPROVED,
270 State.SESSION_ACCEPTED)) {
271 // we might have already changed resources (full jid) at this point; so this might not
272 // even reach the other party
273 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
274 } else {
275 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
276 finish();
277 }
278 }
279
280 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
281 respondOk(jinglePacket);
282 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
283 final State previous = this.state;
284 Log.d(
285 Config.LOGTAG,
286 id.account.getJid().asBareJid()
287 + ": received session terminate reason="
288 + wrapper.reason
289 + "("
290 + Strings.nullToEmpty(wrapper.text)
291 + ") while in state "
292 + previous);
293 if (TERMINATED.contains(previous)) {
294 Log.d(
295 Config.LOGTAG,
296 id.account.getJid().asBareJid()
297 + ": ignoring session terminate because already in "
298 + previous);
299 return;
300 }
301 webRTCWrapper.close();
302 final State target = reasonToState(wrapper.reason);
303 transitionOrThrow(target);
304 writeLogMessage(target);
305 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
306 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
307 }
308 finish();
309 }
310
311 private void receiveTransportInfo(final JinglePacket jinglePacket) {
312 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
313 // INITIALIZED only after transport-info has been received
314 if (isInState(
315 State.NULL,
316 State.PROCEED,
317 State.SESSION_INITIALIZED,
318 State.SESSION_INITIALIZED_PRE_APPROVED,
319 State.SESSION_ACCEPTED)) {
320 final RtpContentMap contentMap;
321 try {
322 contentMap = RtpContentMap.of(jinglePacket);
323 } catch (final IllegalArgumentException | NullPointerException e) {
324 Log.d(
325 Config.LOGTAG,
326 id.account.getJid().asBareJid()
327 + ": improperly formatted contents; ignoring",
328 e);
329 respondOk(jinglePacket);
330 return;
331 }
332 receiveTransportInfo(jinglePacket, contentMap);
333 } else {
334 if (isTerminated()) {
335 respondOk(jinglePacket);
336 Log.d(
337 Config.LOGTAG,
338 id.account.getJid().asBareJid()
339 + ": ignoring out-of-order transport info; we where already terminated");
340 } else {
341 Log.d(
342 Config.LOGTAG,
343 id.account.getJid().asBareJid()
344 + ": received transport info while in state="
345 + this.state);
346 terminateWithOutOfOrder(jinglePacket);
347 }
348 }
349 }
350
351 private void receiveTransportInfo(
352 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
353 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
354 contentMap.contents.entrySet();
355 if (this.state == State.SESSION_ACCEPTED) {
356 // zero candidates + modified credentials are an ICE restart offer
357 if (checkForIceRestart(jinglePacket, contentMap)) {
358 return;
359 }
360 respondOk(jinglePacket);
361 try {
362 processCandidates(candidates);
363 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
364 Log.w(
365 Config.LOGTAG,
366 id.account.getJid().asBareJid()
367 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
368 }
369 } else {
370 respondOk(jinglePacket);
371 pendingIceCandidates.addAll(candidates);
372 }
373 }
374
375 private void receiveContentAdd(final JinglePacket jinglePacket) {
376 // TODO check if in session accepted
377 final RtpContentMap modification;
378 try {
379 modification = RtpContentMap.of(jinglePacket);
380 modification.requireContentDescriptions();
381 } catch (final RuntimeException e) {
382 Log.d(
383 Config.LOGTAG,
384 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
385 Throwables.getRootCause(e));
386 respondOk(jinglePacket);
387 webRTCWrapper.close();
388 sendSessionTerminate(Reason.of(e), e.getMessage());
389 return;
390 }
391 if (isInState(State.SESSION_ACCEPTED)) {
392 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
393 final ListenableFuture<RtpContentMap> future =
394 receiveRtpContentMap(
395 modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
396 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
397 @Override
398 public void onSuccess(final RtpContentMap rtpContentMap) {
399 receiveContentAdd(jinglePacket, rtpContentMap);
400 }
401
402 @Override
403 public void onFailure(@NonNull Throwable throwable) {
404 respondOk(jinglePacket);
405 final Throwable rootCause = Throwables.getRootCause(throwable);
406 Log.d(
407 Config.LOGTAG,
408 id.account.getJid().asBareJid()
409 + ": improperly formatted contents in content-add",
410 throwable);
411 webRTCWrapper.close();
412 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
413 }
414 }, MoreExecutors.directExecutor());
415 } else {
416 terminateWithOutOfOrder(jinglePacket);
417 }
418 }
419
420 private void receiveContentAdd(
421 final JinglePacket jinglePacket, final RtpContentMap modification) {
422 final RtpContentMap remote = getRemoteContentMap();
423 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
424 respondOk(jinglePacket);
425 this.webRTCWrapper.close();
426 sendSessionTerminate(
427 Reason.FAILED_APPLICATION,
428 String.format(
429 "contents with names %s already exists",
430 Joiner.on(", ").join(modification.getNames())));
431 return;
432 }
433 final ContentAddition contentAddition =
434 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
435
436 final RtpContentMap outgoing = this.outgoingContentAdd;
437 final Set<ContentAddition.Summary> outgoingContentAddSummary =
438 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
439
440 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
441 if (isInitiator()) {
442 Log.d(
443 Config.LOGTAG,
444 id.getAccount().getJid().asBareJid()
445 + ": respond with tie break to matching content-add offer");
446 respondWithTieBreak(jinglePacket);
447 } else {
448 Log.d(
449 Config.LOGTAG,
450 id.getAccount().getJid().asBareJid()
451 + ": automatically accept matching content-add offer");
452 acceptContentAdd(contentAddition.summary, modification);
453 }
454 return;
455 }
456
457 // once we can display multiple video tracks we can be more loose with this condition
458 // theoretically it should also be fine to automatically accept audio only contents
459 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
460 Log.d(
461 Config.LOGTAG,
462 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
463 this.incomingContentAdd = modification;
464 respondOk(jinglePacket);
465 updateEndUserState();
466 } else {
467 respondOk(jinglePacket);
468 // TODO do we want to add a reason?
469 rejectContentAdd(modification);
470 }
471 }
472
473 private void receiveContentAccept(final JinglePacket jinglePacket) {
474 final RtpContentMap receivedContentAccept;
475 try {
476 receivedContentAccept = RtpContentMap.of(jinglePacket);
477 receivedContentAccept.requireContentDescriptions();
478 } catch (final RuntimeException e) {
479 Log.d(
480 Config.LOGTAG,
481 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
482 Throwables.getRootCause(e));
483 respondOk(jinglePacket);
484 webRTCWrapper.close();
485 sendSessionTerminate(Reason.of(e), e.getMessage());
486 return;
487 }
488
489 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
490 if (outgoingContentAdd == null) {
491 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
492 terminateWithOutOfOrder(jinglePacket);
493 return;
494 }
495 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
496 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
497 this.outgoingContentAdd = null;
498 respondOk(jinglePacket);
499 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
500 final ListenableFuture<RtpContentMap> future =
501 receiveRtpContentMap(
502 receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
503 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
504 @Override
505 public void onSuccess(final RtpContentMap result) {
506 receiveContentAccept(result);
507 }
508
509 @Override
510 public void onFailure(@NonNull final Throwable throwable) {
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
513 }
514 }, MoreExecutors.directExecutor());
515 } else {
516 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
517 terminateWithOutOfOrder(jinglePacket);
518 }
519 }
520
521 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
522 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
523 final RtpContentMap modifiedContentMap =
524 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
525
526 setRemoteContentMap(modifiedContentMap);
527
528 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
529
530 final org.webrtc.SessionDescription sdp =
531 new org.webrtc.SessionDescription(
532 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
533
534 try {
535 this.webRTCWrapper.setRemoteDescription(sdp).get();
536 } catch (final Exception e) {
537 final Throwable cause = Throwables.getRootCause(e);
538 Log.d(
539 Config.LOGTAG,
540 id.getAccount().getJid().asBareJid()
541 + ": unable to set remote description after receiving content-accept",
542 cause);
543 webRTCWrapper.close();
544 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
545 return;
546 }
547 processCandidates(receivedContentAccept.contents.entrySet());
548 updateEndUserState();
549 Log.d(
550 Config.LOGTAG,
551 id.getAccount().getJid().asBareJid()
552 + ": remote has accepted content-add "
553 + ContentAddition.summary(receivedContentAccept));
554 }
555
556 private void receiveContentModify(final JinglePacket jinglePacket) {
557 // TODO check session accepted
558 final Map<String, Content.Senders> modification =
559 Maps.transformEntries(
560 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
561 final boolean isInitiator = isInitiator();
562 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
563 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
564 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
565 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
566 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
567 respondOk(jinglePacket);
568 final RtpContentMap modifiedContentMap;
569 try {
570 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
571 } catch (final IllegalArgumentException e) {
572 webRTCWrapper.close();
573 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
574 return;
575 }
576 this.outgoingContentAdd = modifiedContentMap;
577 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
578 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
579 respondOk(jinglePacket);
580 final RtpContentMap modifiedRemoteContentMap;
581 try {
582 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
583 } catch (final IllegalArgumentException e) {
584 webRTCWrapper.close();
585 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
586 return;
587 }
588 final SessionDescription offer;
589 try {
590 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
591 } catch (final IllegalArgumentException | NullPointerException e) {
592 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
593 webRTCWrapper.close();
594 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
595 return;
596 }
597 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
598 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
599 } else {
600 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
601 respondWithItemNotFound(jinglePacket);
602 }
603 }
604
605 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
606 this.setRemoteContentMap(modifiedRemoteContentMap);
607 final org.webrtc.SessionDescription sdp =
608 new org.webrtc.SessionDescription(
609 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
610 try {
611 this.webRTCWrapper.setRemoteDescription(sdp).get();
612 // auto accept is only done when we already have tracks
613 final SessionDescription answer = setLocalSessionDescription();
614 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
615 modifyLocalContentMap(rtpContentMap);
616 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
617 //resendCandidatesFromSdp(answer);
618 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
619 } catch (final Exception e) {
620 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
621 webRTCWrapper.close();
622 sendSessionTerminate(Reason.FAILED_APPLICATION);
623 }
624 }
625
626 private void resendCandidatesFromSdp(final SessionDescription answer) {
627 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
628 for(final SessionDescription.Media media : answer.media) {
629 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
630 if (Strings.isNullOrEmpty(mid)) {
631 continue;
632 }
633 for(final String sdpCandidate : media.attributes.get("candidate")) {
634 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
635 if (candidate != null) {
636 candidateBuilder.put(mid,candidate);
637 }
638 }
639 }
640 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
641 sendTransportInfo(candidates);
642 }
643
644 private void receiveContentReject(final JinglePacket jinglePacket) {
645 final RtpContentMap receivedContentReject;
646 try {
647 receivedContentReject = RtpContentMap.of(jinglePacket);
648 } catch (final RuntimeException e) {
649 Log.d(
650 Config.LOGTAG,
651 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
652 Throwables.getRootCause(e));
653 respondOk(jinglePacket);
654 this.webRTCWrapper.close();
655 sendSessionTerminate(Reason.of(e), e.getMessage());
656 return;
657 }
658
659 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
660 if (outgoingContentAdd == null) {
661 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
662 terminateWithOutOfOrder(jinglePacket);
663 return;
664 }
665 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
666 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
667 this.outgoingContentAdd = null;
668 respondOk(jinglePacket);
669 Log.d(Config.LOGTAG,jinglePacket.toString());
670 receiveContentReject(ourSummary);
671 } else {
672 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
673 terminateWithOutOfOrder(jinglePacket);
674 }
675 }
676
677 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
678 try {
679 this.webRTCWrapper.removeTrack(Media.VIDEO);
680 final RtpContentMap localContentMap = customRollback();
681 modifyLocalContentMap(localContentMap);
682 } catch (final Exception e) {
683 final Throwable cause = Throwables.getRootCause(e);
684 Log.d(
685 Config.LOGTAG,
686 id.getAccount().getJid().asBareJid()
687 + ": unable to rollback local description after receiving content-reject",
688 cause);
689 webRTCWrapper.close();
690 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
691 return;
692 }
693 Log.d(
694 Config.LOGTAG,
695 id.getAccount().getJid().asBareJid()
696 + ": remote has rejected our content-add "
697 + summary);
698 }
699
700 private void receiveContentRemove(final JinglePacket jinglePacket) {
701 final RtpContentMap receivedContentRemove;
702 try {
703 receivedContentRemove = RtpContentMap.of(jinglePacket);
704 receivedContentRemove.requireContentDescriptions();
705 } catch (final RuntimeException e) {
706 Log.d(
707 Config.LOGTAG,
708 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
709 Throwables.getRootCause(e));
710 respondOk(jinglePacket);
711 this.webRTCWrapper.close();
712 sendSessionTerminate(Reason.of(e), e.getMessage());
713 return;
714 }
715 respondOk(jinglePacket);
716 receiveContentRemove(receivedContentRemove);
717 }
718
719 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
720 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
721 final Set<ContentAddition.Summary> contentAddSummary =
722 incomingContentAdd == null
723 ? Collections.emptySet()
724 : ContentAddition.summary(incomingContentAdd);
725 final Set<ContentAddition.Summary> removeSummary =
726 ContentAddition.summary(receivedContentRemove);
727 if (contentAddSummary.equals(removeSummary)) {
728 this.incomingContentAdd = null;
729 updateEndUserState();
730 } else {
731 webRTCWrapper.close();
732 sendSessionTerminate(
733 Reason.FAILED_APPLICATION,
734 String.format(
735 "%s only supports %s as a means to retract a not yet accepted %s",
736 BuildConfig.APP_NAME,
737 JinglePacket.Action.CONTENT_REMOVE,
738 JinglePacket.Action.CONTENT_ADD));
739 }
740 }
741
742 public synchronized void retractContentAdd() {
743 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
744 if (outgoingContentAdd == null) {
745 throw new IllegalStateException("Not outgoing content add");
746 }
747 try {
748 webRTCWrapper.removeTrack(Media.VIDEO);
749 final RtpContentMap localContentMap = customRollback();
750 modifyLocalContentMap(localContentMap);
751 } catch (final Exception e) {
752 final Throwable cause = Throwables.getRootCause(e);
753 Log.d(
754 Config.LOGTAG,
755 id.getAccount().getJid().asBareJid()
756 + ": unable to rollback local description after trying to retract content-add",
757 cause);
758 webRTCWrapper.close();
759 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
760 return;
761 }
762 this.outgoingContentAdd = null;
763 final JinglePacket retract =
764 outgoingContentAdd
765 .toStub()
766 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
767 this.send(retract);
768 Log.d(
769 Config.LOGTAG,
770 id.getAccount().getJid()
771 + ": retract content-add "
772 + ContentAddition.summary(outgoingContentAdd));
773 }
774
775 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
776 final SessionDescription sdp = setLocalSessionDescription();
777 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
778 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
779 this.webRTCWrapper
780 .setRemoteDescription(
781 new org.webrtc.SessionDescription(
782 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
783 .get();
784 return localRtpContentMap;
785 }
786
787 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
788 final RtpContentMap currentRemote = getRemoteContentMap();
789 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
790 if (diff.isEmpty()) {
791 throw new IllegalStateException(
792 "Unexpected rollback condition. No difference between local and remote");
793 }
794 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
795 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
796 final RtpContentMap nextRemote =
797 currentRemote.addContent(
798 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
799 return SessionDescription.of(nextRemote, !isInitiator());
800 }
801 throw new IllegalStateException(
802 "Unexpected rollback condition. Senders were not uniformly none");
803 }
804
805 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
806 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
807 if (incomingContentAdd == null) {
808 throw new IllegalStateException("No incoming content add");
809 }
810
811 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
812 this.incomingContentAdd = null;
813 acceptContentAdd(contentAddition, incomingContentAdd);
814 } else {
815 throw new IllegalStateException("Accepted content add does not match pending content-add");
816 }
817 }
818
819 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
820 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
821 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
822 this.setRemoteContentMap(modifiedContentMap);
823
824 final SessionDescription offer;
825 try {
826 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
827 } catch (final IllegalArgumentException | NullPointerException e) {
828 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
829 webRTCWrapper.close();
830 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
831 return;
832 }
833 this.incomingContentAdd = null;
834 acceptContentAdd(contentAddition, offer);
835 }
836
837 private void acceptContentAdd(
838 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
839 final org.webrtc.SessionDescription sdp =
840 new org.webrtc.SessionDescription(
841 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
842 try {
843 this.webRTCWrapper.setRemoteDescription(sdp).get();
844
845 // TODO add tracks for 'media' where contentAddition.senders matches
846
847 // TODO if senders.sending(isInitiator())
848
849 this.webRTCWrapper.addTrack(Media.VIDEO);
850
851 // TODO add additional transceivers for recv only cases
852
853 final SessionDescription answer = setLocalSessionDescription();
854 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
855
856 final RtpContentMap contentAcceptMap =
857 rtpContentMap.toContentModification(
858 Collections2.transform(contentAddition, ca -> ca.name));
859
860 Log.d(
861 Config.LOGTAG,
862 id.getAccount().getJid().asBareJid()
863 + ": sending content-accept "
864 + ContentAddition.summary(contentAcceptMap));
865 modifyLocalContentMap(rtpContentMap);
866 final ListenableFuture<RtpContentMap> future = prepareOutgoingContentMap(contentAcceptMap);
867 Futures.addCallback(
868 future,
869 new FutureCallback<RtpContentMap>() {
870 @Override
871 public void onSuccess(final RtpContentMap rtpContentMap) {
872 sendContentAccept(rtpContentMap);
873 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
874 }
875
876 @Override
877 public void onFailure(@NonNull final Throwable throwable) {
878 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
879 }
880 },
881 MoreExecutors.directExecutor());
882 } catch (final Exception e) {
883 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
884 webRTCWrapper.close();
885 sendSessionTerminate(Reason.FAILED_APPLICATION);
886 }
887 }
888
889 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
890 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
891 send(jinglePacket);
892 }
893
894 public synchronized void rejectContentAdd() {
895 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
896 if (incomingContentAdd == null) {
897 throw new IllegalStateException("No incoming content add");
898 }
899 this.incomingContentAdd = null;
900 updateEndUserState();
901 rejectContentAdd(incomingContentAdd);
902 }
903
904 private void rejectContentAdd(final RtpContentMap contentMap) {
905 final JinglePacket jinglePacket =
906 contentMap
907 .toStub()
908 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
909 Log.d(
910 Config.LOGTAG,
911 id.getAccount().getJid().asBareJid()
912 + ": rejecting content "
913 + ContentAddition.summary(contentMap));
914 send(jinglePacket);
915 }
916
917 private boolean checkForIceRestart(
918 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
919 final RtpContentMap existing = getRemoteContentMap();
920 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
921 final IceUdpTransportInfo.Credentials newCredentials;
922 try {
923 existingCredentials = existing.getCredentials();
924 newCredentials = rtpContentMap.getDistinctCredentials();
925 } catch (final IllegalStateException e) {
926 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
927 return false;
928 }
929 if (existingCredentials.contains(newCredentials)) {
930 return false;
931 }
932 // TODO an alternative approach is to check if we already got an iq result to our
933 // ICE-restart
934 // and if that's the case we are seeing an answer.
935 // This might be more spec compliant but also more error prone potentially
936 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
937 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
938 final boolean isOffer = rtpContentMap.emptyCandidates();
939 final RtpContentMap restartContentMap;
940 try {
941 if (isOffer) {
942 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
943 restartContentMap =
944 existing.modifiedCredentials(
945 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
946 } else {
947 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
948 Log.d(
949 Config.LOGTAG,
950 "received confirmation of ICE restart"
951 + newCredentials
952 + " peer_setup="
953 + setup);
954 // DTLS setup attribute needs to be rewritten to reflect current peer state
955 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
956 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
957 }
958 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
959 return isOffer;
960 } else {
961 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
962 respondWithTieBreak(jinglePacket);
963 return true;
964 }
965 } catch (final Exception exception) {
966 respondOk(jinglePacket);
967 final Throwable rootCause = Throwables.getRootCause(exception);
968 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
969 // If this happens a termination is already in progress
970 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
971 return true;
972 }
973 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
974 webRTCWrapper.close();
975 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
976 return true;
977 }
978 }
979
980 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
981 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
982 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
983 throw new IllegalStateException("Invalid peer setup");
984 }
985 return peerSetup;
986 }
987
988 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
989 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
990 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
991 }
992 this.peerDtlsSetup = setup;
993 }
994
995 private boolean applyIceRestart(
996 final JinglePacket jinglePacket,
997 final RtpContentMap restartContentMap,
998 final boolean isOffer)
999 throws ExecutionException, InterruptedException {
1000 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
1001 final org.webrtc.SessionDescription.Type type =
1002 isOffer
1003 ? org.webrtc.SessionDescription.Type.OFFER
1004 : org.webrtc.SessionDescription.Type.ANSWER;
1005 org.webrtc.SessionDescription sdp =
1006 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1007 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1008 if (isInitiator()) {
1009 // We ignore the offer and respond with tie-break. This will clause the responder
1010 // not to apply the content map
1011 return false;
1012 }
1013 }
1014 webRTCWrapper.setRemoteDescription(sdp).get();
1015 setRemoteContentMap(restartContentMap);
1016 if (isOffer) {
1017 final SessionDescription localSessionDescription = setLocalSessionDescription();
1018 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1019 // We need to respond OK before sending any candidates
1020 respondOk(jinglePacket);
1021 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1022 } else {
1023 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1024 }
1025 return true;
1026 }
1027
1028 private void processCandidates(
1029 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1030 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1031 processCandidate(content);
1032 }
1033 }
1034
1035 private void processCandidate(
1036 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1037 final RtpContentMap rtpContentMap = getRemoteContentMap();
1038 final List<String> indices = toIdentificationTags(rtpContentMap);
1039 final String sdpMid = content.getKey(); // aka content name
1040 final IceUdpTransportInfo transport = content.getValue().transport;
1041 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1042
1043 // TODO check that credentials remained the same
1044
1045 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1046 final String sdp;
1047 try {
1048 sdp = candidate.toSdpAttribute(credentials.ufrag);
1049 } catch (final IllegalArgumentException e) {
1050 Log.d(
1051 Config.LOGTAG,
1052 id.account.getJid().asBareJid()
1053 + ": ignoring invalid ICE candidate "
1054 + e.getMessage());
1055 continue;
1056 }
1057 final int mLineIndex = indices.indexOf(sdpMid);
1058 if (mLineIndex < 0) {
1059 Log.w(
1060 Config.LOGTAG,
1061 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1062 }
1063 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1064 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1065 this.webRTCWrapper.addIceCandidate(iceCandidate);
1066 }
1067 }
1068
1069 private RtpContentMap getRemoteContentMap() {
1070 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1071 }
1072
1073 private RtpContentMap getLocalContentMap() {
1074 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1075 }
1076
1077 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1078 final Group originalGroup = rtpContentMap.group;
1079 final List<String> identificationTags =
1080 originalGroup == null
1081 ? rtpContentMap.getNames()
1082 : originalGroup.getIdentificationTags();
1083 if (identificationTags.size() == 0) {
1084 Log.w(
1085 Config.LOGTAG,
1086 id.account.getJid().asBareJid()
1087 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1088 }
1089 return identificationTags;
1090 }
1091
1092 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1093 final JinglePacket jinglePacket, final boolean expectVerification) {
1094 try {
1095 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1096 } catch (final Exception e) {
1097 return Futures.immediateFailedFuture(e);
1098 }
1099 }
1100 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) {
1101 Log.d(
1102 Config.LOGTAG,
1103 "receiveRtpContentMap("
1104 + receivedContentMap.getClass().getSimpleName()
1105 + ",expectVerification="
1106 + expectVerification
1107 + ")");
1108 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1109 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1110 id.account
1111 .getAxolotlService()
1112 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1113 return Futures.transform(
1114 future,
1115 omemoVerifiedPayload -> {
1116 // TODO test if an exception here triggers a correct abort
1117 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1118 Log.d(
1119 Config.LOGTAG,
1120 id.account.getJid().asBareJid()
1121 + ": received verifiable DTLS fingerprint via "
1122 + omemoVerification);
1123 return omemoVerifiedPayload.getPayload();
1124 },
1125 MoreExecutors.directExecutor());
1126 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1127 return Futures.immediateFailedFuture(
1128 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1129 } else {
1130 return Futures.immediateFuture(receivedContentMap);
1131 }
1132 }
1133
1134 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1135 if (isInitiator()) {
1136 Log.d(
1137 Config.LOGTAG,
1138 String.format(
1139 "%s: received session-initiate even though we were initiating",
1140 id.account.getJid().asBareJid()));
1141 if (isTerminated()) {
1142 Log.d(
1143 Config.LOGTAG,
1144 String.format(
1145 "%s: got a reason to terminate with out-of-order. but already in state %s",
1146 id.account.getJid().asBareJid(), getState()));
1147 respondWithOutOfOrder(jinglePacket);
1148 } else {
1149 terminateWithOutOfOrder(jinglePacket);
1150 }
1151 return;
1152 }
1153 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1154 Futures.addCallback(
1155 future,
1156 new FutureCallback<RtpContentMap>() {
1157 @Override
1158 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1159 receiveSessionInitiate(jinglePacket, rtpContentMap);
1160 }
1161
1162 @Override
1163 public void onFailure(@NonNull final Throwable throwable) {
1164 respondOk(jinglePacket);
1165 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1166 }
1167 },
1168 MoreExecutors.directExecutor());
1169 }
1170
1171 private void receiveSessionInitiate(
1172 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1173 try {
1174 contentMap.requireContentDescriptions();
1175 contentMap.requireDTLSFingerprint(true);
1176 } catch (final RuntimeException e) {
1177 Log.d(
1178 Config.LOGTAG,
1179 id.account.getJid().asBareJid() + ": improperly formatted contents",
1180 Throwables.getRootCause(e));
1181 respondOk(jinglePacket);
1182 sendSessionTerminate(Reason.of(e), e.getMessage());
1183 return;
1184 }
1185 Log.d(
1186 Config.LOGTAG,
1187 "processing session-init with " + contentMap.contents.size() + " contents");
1188 final State target;
1189 if (this.state == State.PROCEED) {
1190 Preconditions.checkState(
1191 proposedMedia != null && proposedMedia.size() > 0,
1192 "proposed media must be set when processing pre-approved session-initiate");
1193 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1194 sendSessionTerminate(
1195 Reason.SECURITY_ERROR,
1196 String.format(
1197 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1198 this.proposedMedia, contentMap.getMedia()));
1199 return;
1200 }
1201 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1202 } else {
1203 target = State.SESSION_INITIALIZED;
1204 }
1205 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1206 respondOk(jinglePacket);
1207 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1208 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1209 Log.d(
1210 Config.LOGTAG,
1211 id.account.getJid().asBareJid()
1212 + ": automatically accepting session-initiate");
1213 sendSessionAccept();
1214 } else {
1215 Log.d(
1216 Config.LOGTAG,
1217 id.account.getJid().asBareJid()
1218 + ": received not pre-approved session-initiate. start ringing");
1219 startRinging();
1220 }
1221 } else {
1222 Log.d(
1223 Config.LOGTAG,
1224 String.format(
1225 "%s: received session-initiate while in state %s",
1226 id.account.getJid().asBareJid(), state));
1227 terminateWithOutOfOrder(jinglePacket);
1228 }
1229 }
1230
1231 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1232 if (!isInitiator()) {
1233 Log.d(
1234 Config.LOGTAG,
1235 String.format(
1236 "%s: received session-accept even though we were responding",
1237 id.account.getJid().asBareJid()));
1238 terminateWithOutOfOrder(jinglePacket);
1239 return;
1240 }
1241 final ListenableFuture<RtpContentMap> future =
1242 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1243 Futures.addCallback(
1244 future,
1245 new FutureCallback<RtpContentMap>() {
1246 @Override
1247 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1248 receiveSessionAccept(jinglePacket, rtpContentMap);
1249 }
1250
1251 @Override
1252 public void onFailure(@NonNull final Throwable throwable) {
1253 respondOk(jinglePacket);
1254 Log.d(
1255 Config.LOGTAG,
1256 id.account.getJid().asBareJid()
1257 + ": improperly formatted contents in session-accept",
1258 throwable);
1259 webRTCWrapper.close();
1260 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1261 }
1262 },
1263 MoreExecutors.directExecutor());
1264 }
1265
1266 private void receiveSessionAccept(
1267 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1268 try {
1269 contentMap.requireContentDescriptions();
1270 contentMap.requireDTLSFingerprint();
1271 } catch (final RuntimeException e) {
1272 respondOk(jinglePacket);
1273 Log.d(
1274 Config.LOGTAG,
1275 id.account.getJid().asBareJid()
1276 + ": improperly formatted contents in session-accept",
1277 e);
1278 webRTCWrapper.close();
1279 sendSessionTerminate(Reason.of(e), e.getMessage());
1280 return;
1281 }
1282 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1283 if (!initiatorMedia.equals(contentMap.getMedia())) {
1284 sendSessionTerminate(
1285 Reason.SECURITY_ERROR,
1286 String.format(
1287 "Your session-included included media %s but our session-initiate was %s",
1288 this.proposedMedia, contentMap.getMedia()));
1289 return;
1290 }
1291 Log.d(
1292 Config.LOGTAG,
1293 "processing session-accept with " + contentMap.contents.size() + " contents");
1294 if (transition(State.SESSION_ACCEPTED)) {
1295 respondOk(jinglePacket);
1296 receiveSessionAccept(contentMap);
1297 } else {
1298 Log.d(
1299 Config.LOGTAG,
1300 String.format(
1301 "%s: received session-accept while in state %s",
1302 id.account.getJid().asBareJid(), state));
1303 respondOk(jinglePacket);
1304 }
1305 }
1306
1307 private void receiveSessionAccept(final RtpContentMap contentMap) {
1308 this.responderRtpContentMap = contentMap;
1309 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1310 final SessionDescription sessionDescription;
1311 try {
1312 sessionDescription = SessionDescription.of(contentMap, false);
1313 } catch (final IllegalArgumentException | NullPointerException e) {
1314 Log.d(
1315 Config.LOGTAG,
1316 id.account.getJid().asBareJid()
1317 + ": unable convert offer from session-accept to SDP",
1318 e);
1319 webRTCWrapper.close();
1320 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1321 return;
1322 }
1323 final org.webrtc.SessionDescription answer =
1324 new org.webrtc.SessionDescription(
1325 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1326 try {
1327 this.webRTCWrapper.setRemoteDescription(answer).get();
1328 } catch (final Exception e) {
1329 Log.d(
1330 Config.LOGTAG,
1331 id.account.getJid().asBareJid()
1332 + ": unable to set remote description after receiving session-accept",
1333 Throwables.getRootCause(e));
1334 webRTCWrapper.close();
1335 sendSessionTerminate(
1336 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1337 return;
1338 }
1339 processCandidates(contentMap.contents.entrySet());
1340 }
1341
1342 private void sendSessionAccept() {
1343 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1344 if (rtpContentMap == null) {
1345 throw new IllegalStateException("initiator RTP Content Map has not been set");
1346 }
1347 final SessionDescription offer;
1348 try {
1349 offer = SessionDescription.of(rtpContentMap, true);
1350 } catch (final IllegalArgumentException | NullPointerException e) {
1351 Log.d(
1352 Config.LOGTAG,
1353 id.account.getJid().asBareJid()
1354 + ": unable convert offer from session-initiate to SDP",
1355 e);
1356 webRTCWrapper.close();
1357 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1358 return;
1359 }
1360 sendSessionAccept(rtpContentMap.getMedia(), offer);
1361 }
1362
1363 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1364 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1365 }
1366
1367 private synchronized void sendSessionAccept(
1368 final Set<Media> media,
1369 final SessionDescription offer,
1370 final List<PeerConnection.IceServer> iceServers) {
1371 if (isTerminated()) {
1372 Log.w(
1373 Config.LOGTAG,
1374 id.account.getJid().asBareJid()
1375 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1376 return;
1377 }
1378 try {
1379 setupWebRTC(media, iceServers);
1380 } catch (final WebRTCWrapper.InitializationException e) {
1381 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1382 webRTCWrapper.close();
1383 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1384 return;
1385 }
1386 final org.webrtc.SessionDescription sdp =
1387 new org.webrtc.SessionDescription(
1388 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1389 try {
1390 this.webRTCWrapper.setRemoteDescription(sdp).get();
1391 addIceCandidatesFromBlackLog();
1392 org.webrtc.SessionDescription webRTCSessionDescription =
1393 this.webRTCWrapper.setLocalDescription().get();
1394 prepareSessionAccept(webRTCSessionDescription);
1395 } catch (final Exception e) {
1396 failureToAcceptSession(e);
1397 }
1398 }
1399
1400 private void failureToAcceptSession(final Throwable throwable) {
1401 if (isTerminated()) {
1402 return;
1403 }
1404 final Throwable rootCause = Throwables.getRootCause(throwable);
1405 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1406 webRTCWrapper.close();
1407 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1408 }
1409
1410 private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) {
1411 if (isTerminated()) {
1412 return;
1413 }
1414 final Throwable rootCause = Throwables.getRootCause(throwable);
1415 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1416 webRTCWrapper.close();
1417 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1418 }
1419
1420 private void addIceCandidatesFromBlackLog() {
1421 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1422 while ((foo = this.pendingIceCandidates.poll()) != null) {
1423 processCandidate(foo);
1424 Log.d(
1425 Config.LOGTAG,
1426 id.account.getJid().asBareJid() + ": added candidate from back log");
1427 }
1428 }
1429
1430 private void prepareSessionAccept(
1431 final org.webrtc.SessionDescription webRTCSessionDescription) {
1432 final SessionDescription sessionDescription =
1433 SessionDescription.parse(webRTCSessionDescription.description);
1434 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1435 this.responderRtpContentMap = respondingRtpContentMap;
1436 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1437 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1438 prepareOutgoingContentMap(respondingRtpContentMap);
1439 Futures.addCallback(
1440 outgoingContentMapFuture,
1441 new FutureCallback<RtpContentMap>() {
1442 @Override
1443 public void onSuccess(final RtpContentMap outgoingContentMap) {
1444 sendSessionAccept(outgoingContentMap);
1445 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1446 }
1447
1448 @Override
1449 public void onFailure(@NonNull Throwable throwable) {
1450 failureToAcceptSession(throwable);
1451 }
1452 },
1453 MoreExecutors.directExecutor());
1454 }
1455
1456 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1457 if (isTerminated()) {
1458 Log.w(
1459 Config.LOGTAG,
1460 id.account.getJid().asBareJid()
1461 + ": preparing session accept was too slow. already terminated. nothing to do.");
1462 return;
1463 }
1464 transitionOrThrow(State.SESSION_ACCEPTED);
1465 final JinglePacket sessionAccept =
1466 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1467 send(sessionAccept);
1468 }
1469
1470 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1471 final RtpContentMap rtpContentMap) {
1472 if (this.omemoVerification.hasDeviceId()) {
1473 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1474 verifiedPayloadFuture =
1475 id.account
1476 .getAxolotlService()
1477 .encrypt(
1478 rtpContentMap,
1479 id.with,
1480 omemoVerification.getDeviceId());
1481 return Futures.transform(
1482 verifiedPayloadFuture,
1483 verifiedPayload -> {
1484 omemoVerification.setOrEnsureEqual(verifiedPayload);
1485 return verifiedPayload.getPayload();
1486 },
1487 MoreExecutors.directExecutor());
1488 } else {
1489 return Futures.immediateFuture(rtpContentMap);
1490 }
1491 }
1492
1493 synchronized void deliveryMessage(
1494 final Jid from,
1495 final Element message,
1496 final String serverMessageId,
1497 final long timestamp) {
1498 Log.d(
1499 Config.LOGTAG,
1500 id.account.getJid().asBareJid()
1501 + ": delivered message to JingleRtpConnection "
1502 + message);
1503 switch (message.getName()) {
1504 case "propose":
1505 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1506 break;
1507 case "proceed":
1508 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1509 break;
1510 case "retract":
1511 receiveRetract(from, serverMessageId, timestamp);
1512 break;
1513 case "reject":
1514 receiveReject(from, serverMessageId, timestamp);
1515 break;
1516 case "accept":
1517 receiveAccept(from, serverMessageId, timestamp);
1518 break;
1519 default:
1520 break;
1521 }
1522 }
1523
1524 void deliverFailedProceed(final String message) {
1525 Log.d(
1526 Config.LOGTAG,
1527 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1528 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1529 webRTCWrapper.close();
1530 Log.d(
1531 Config.LOGTAG,
1532 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1533 this.finish();
1534 }
1535 }
1536
1537 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1538 final boolean originatedFromMyself =
1539 from.asBareJid().equals(id.account.getJid().asBareJid());
1540 if (originatedFromMyself) {
1541 if (transition(State.ACCEPTED)) {
1542 acceptedOnOtherDevice(serverMsgId, timestamp);
1543 } else {
1544 Log.d(
1545 Config.LOGTAG,
1546 id.account.getJid().asBareJid()
1547 + ": unable to transition to accept because already in state="
1548 + this.state);
1549 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1550 }
1551 } else {
1552 Log.d(
1553 Config.LOGTAG,
1554 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1555 }
1556 }
1557
1558 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1559 if (serverMsgId != null) {
1560 this.message.setServerMsgId(serverMsgId);
1561 }
1562 this.message.setTime(timestamp);
1563 this.message.setCarbon(true); // indicate that call was accepted on other device
1564 this.writeLogMessageSuccess(0);
1565 this.xmppConnectionService
1566 .getNotificationService()
1567 .cancelIncomingCallNotification();
1568 this.finish();
1569 }
1570
1571 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1572 final boolean originatedFromMyself =
1573 from.asBareJid().equals(id.account.getJid().asBareJid());
1574 // reject from another one of my clients
1575 if (originatedFromMyself) {
1576 receiveRejectFromMyself(serverMsgId, timestamp);
1577 } else if (isInitiator()) {
1578 if (from.equals(id.with)) {
1579 receiveRejectFromResponder();
1580 } else {
1581 Log.d(
1582 Config.LOGTAG,
1583 id.account.getJid()
1584 + ": ignoring reject from "
1585 + from
1586 + " for session with "
1587 + id.with);
1588 }
1589 } else {
1590 Log.d(
1591 Config.LOGTAG,
1592 id.account.getJid()
1593 + ": ignoring reject from "
1594 + from
1595 + " for session with "
1596 + id.with);
1597 }
1598 }
1599
1600 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1601 if (transition(State.REJECTED)) {
1602 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1603 this.finish();
1604 if (serverMsgId != null) {
1605 this.message.setServerMsgId(serverMsgId);
1606 }
1607 this.message.setTime(timestamp);
1608 this.message.setCarbon(true); // indicate that call was rejected on other device
1609 writeLogMessageMissed();
1610 } else {
1611 Log.d(
1612 Config.LOGTAG,
1613 "not able to transition into REJECTED because already in " + this.state);
1614 }
1615 }
1616
1617 private void receiveRejectFromResponder() {
1618 if (isInState(State.PROCEED)) {
1619 Log.d(
1620 Config.LOGTAG,
1621 id.account.getJid()
1622 + ": received reject while still in proceed. callee reconsidered");
1623 closeTransitionLogFinish(State.REJECTED_RACED);
1624 return;
1625 }
1626 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1627 Log.d(
1628 Config.LOGTAG,
1629 id.account.getJid()
1630 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1631 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1632 return;
1633 }
1634 Log.d(
1635 Config.LOGTAG,
1636 id.account.getJid()
1637 + ": ignoring reject from responder because already in state "
1638 + this.state);
1639 }
1640
1641 private void receivePropose(
1642 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1643 final boolean originatedFromMyself =
1644 from.asBareJid().equals(id.account.getJid().asBareJid());
1645 if (originatedFromMyself) {
1646 Log.d(
1647 Config.LOGTAG,
1648 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1649 } else if (transition(
1650 State.PROPOSED,
1651 () -> {
1652 final Collection<RtpDescription> descriptions =
1653 Collections2.transform(
1654 Collections2.filter(
1655 propose.getDescriptions(),
1656 d -> d instanceof RtpDescription),
1657 input -> (RtpDescription) input);
1658 final Collection<Media> media =
1659 Collections2.transform(descriptions, RtpDescription::getMedia);
1660 Preconditions.checkState(
1661 !media.contains(Media.UNKNOWN),
1662 "RTP descriptions contain unknown media");
1663 Log.d(
1664 Config.LOGTAG,
1665 id.account.getJid().asBareJid()
1666 + ": received session proposal from "
1667 + from
1668 + " for "
1669 + media);
1670 this.proposedMedia = Sets.newHashSet(media);
1671 })) {
1672 if (serverMsgId != null) {
1673 this.message.setServerMsgId(serverMsgId);
1674 }
1675 this.message.setTime(timestamp);
1676 startRinging();
1677 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1678 sendJingleMessage("ringing");
1679 }
1680 } else {
1681 Log.d(
1682 Config.LOGTAG,
1683 id.account.getJid()
1684 + ": ignoring session proposal because already in "
1685 + state);
1686 }
1687 }
1688
1689 private void startRinging() {
1690 Log.d(
1691 Config.LOGTAG,
1692 id.account.getJid().asBareJid()
1693 + ": received call from "
1694 + id.with
1695 + ". start ringing");
1696 ringingTimeoutFuture =
1697 jingleConnectionManager.schedule(
1698 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1699 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1700 }
1701
1702 private synchronized void ringingTimeout() {
1703 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1704 switch (this.state) {
1705 case PROPOSED:
1706 message.markUnread();
1707 rejectCallFromProposed();
1708 break;
1709 case SESSION_INITIALIZED:
1710 message.markUnread();
1711 rejectCallFromSessionInitiate();
1712 break;
1713 }
1714 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1715 }
1716
1717 private void cancelRingingTimeout() {
1718 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1719 if (future != null && !future.isCancelled()) {
1720 future.cancel(false);
1721 }
1722 }
1723
1724 private void receiveProceed(
1725 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1726 final Set<Media> media =
1727 Preconditions.checkNotNull(
1728 this.proposedMedia, "Proposed media has to be set before handling proceed");
1729 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1730 if (from.equals(id.with)) {
1731 if (isInitiator()) {
1732 if (transition(State.PROCEED)) {
1733 if (serverMsgId != null) {
1734 this.message.setServerMsgId(serverMsgId);
1735 }
1736 this.message.setTime(timestamp);
1737 final Integer remoteDeviceId = proceed.getDeviceId();
1738 if (isOmemoEnabled()) {
1739 this.omemoVerification.setDeviceId(remoteDeviceId);
1740 } else {
1741 if (remoteDeviceId != null) {
1742 Log.d(
1743 Config.LOGTAG,
1744 id.account.getJid().asBareJid()
1745 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1746 }
1747 this.omemoVerification.setDeviceId(null);
1748 }
1749 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1750 } else {
1751 Log.d(
1752 Config.LOGTAG,
1753 String.format(
1754 "%s: ignoring proceed because already in %s",
1755 id.account.getJid().asBareJid(), this.state));
1756 }
1757 } else {
1758 Log.d(
1759 Config.LOGTAG,
1760 String.format(
1761 "%s: ignoring proceed because we were not initializing",
1762 id.account.getJid().asBareJid()));
1763 }
1764 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1765 if (transition(State.ACCEPTED)) {
1766 Log.d(
1767 Config.LOGTAG,
1768 id.account.getJid().asBareJid()
1769 + ": moved session with "
1770 + id.with
1771 + " into state accepted after received carbon copied proceed");
1772 acceptedOnOtherDevice(serverMsgId, timestamp);
1773 }
1774 } else {
1775 Log.d(
1776 Config.LOGTAG,
1777 String.format(
1778 "%s: ignoring proceed from %s. was expected from %s",
1779 id.account.getJid().asBareJid(), from, id.with));
1780 }
1781 }
1782
1783 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1784 if (from.equals(id.with)) {
1785 final State target =
1786 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1787 if (transition(target)) {
1788 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1789 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1790 Log.d(
1791 Config.LOGTAG,
1792 id.account.getJid().asBareJid()
1793 + ": session with "
1794 + id.with
1795 + " has been retracted (serverMsgId="
1796 + serverMsgId
1797 + ")");
1798 if (serverMsgId != null) {
1799 this.message.setServerMsgId(serverMsgId);
1800 }
1801 this.message.setTime(timestamp);
1802 if (target == State.RETRACTED) {
1803 this.message.markUnread();
1804 }
1805 writeLogMessageMissed();
1806 finish();
1807 } else {
1808 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1809 }
1810 } else {
1811 // TODO parse retract from self
1812 Log.d(
1813 Config.LOGTAG,
1814 id.account.getJid().asBareJid()
1815 + ": received retract from "
1816 + from
1817 + ". expected retract from"
1818 + id.with
1819 + ". ignoring");
1820 }
1821 }
1822
1823 public void sendSessionInitiate() {
1824 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1825 }
1826
1827 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1828 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1829 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1830 }
1831
1832 private synchronized void sendSessionInitiate(
1833 final Set<Media> media,
1834 final State targetState,
1835 final List<PeerConnection.IceServer> iceServers) {
1836 if (isTerminated()) {
1837 Log.w(
1838 Config.LOGTAG,
1839 id.account.getJid().asBareJid()
1840 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1841 return;
1842 }
1843 try {
1844 setupWebRTC(media, iceServers);
1845 } catch (final WebRTCWrapper.InitializationException e) {
1846 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1847 webRTCWrapper.close();
1848 sendRetract(Reason.ofThrowable(e));
1849 return;
1850 }
1851 try {
1852 org.webrtc.SessionDescription webRTCSessionDescription =
1853 this.webRTCWrapper.setLocalDescription().get();
1854 prepareSessionInitiate(webRTCSessionDescription, targetState);
1855 } catch (final Exception e) {
1856 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1857 // exceptions
1858 failureToInitiateSession(e, targetState);
1859 }
1860 }
1861
1862 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1863 if (isTerminated()) {
1864 return;
1865 }
1866 Log.d(
1867 Config.LOGTAG,
1868 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1869 Throwables.getRootCause(throwable));
1870 webRTCWrapper.close();
1871 final Reason reason = Reason.ofThrowable(throwable);
1872 if (isInState(targetState)) {
1873 sendSessionTerminate(reason, throwable.getMessage());
1874 } else {
1875 sendRetract(reason);
1876 }
1877 }
1878
1879 private void sendRetract(final Reason reason) {
1880 // TODO embed reason into retract
1881 sendJingleMessage("retract", id.with.asBareJid());
1882 transitionOrThrow(reasonToState(reason));
1883 this.finish();
1884 }
1885
1886 private void prepareSessionInitiate(
1887 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1888 final SessionDescription sessionDescription =
1889 SessionDescription.parse(webRTCSessionDescription.description);
1890 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1891 this.initiatorRtpContentMap = rtpContentMap;
1892 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1893 encryptSessionInitiate(rtpContentMap);
1894 Futures.addCallback(
1895 outgoingContentMapFuture,
1896 new FutureCallback<RtpContentMap>() {
1897 @Override
1898 public void onSuccess(final RtpContentMap outgoingContentMap) {
1899 sendSessionInitiate(outgoingContentMap, targetState);
1900 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1901 }
1902
1903 @Override
1904 public void onFailure(@NonNull final Throwable throwable) {
1905 failureToInitiateSession(throwable, targetState);
1906 }
1907 },
1908 MoreExecutors.directExecutor());
1909 }
1910
1911 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1912 if (isTerminated()) {
1913 Log.w(
1914 Config.LOGTAG,
1915 id.account.getJid().asBareJid()
1916 + ": preparing session was too slow. already terminated. nothing to do.");
1917 return;
1918 }
1919 this.transitionOrThrow(targetState);
1920 final JinglePacket sessionInitiate =
1921 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1922 send(sessionInitiate);
1923 }
1924
1925 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1926 final RtpContentMap rtpContentMap) {
1927 if (this.omemoVerification.hasDeviceId()) {
1928 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1929 verifiedPayloadFuture =
1930 id.account
1931 .getAxolotlService()
1932 .encrypt(
1933 rtpContentMap,
1934 id.with,
1935 omemoVerification.getDeviceId());
1936 final ListenableFuture<RtpContentMap> future =
1937 Futures.transform(
1938 verifiedPayloadFuture,
1939 verifiedPayload -> {
1940 omemoVerification.setSessionFingerprint(
1941 verifiedPayload.getFingerprint());
1942 return verifiedPayload.getPayload();
1943 },
1944 MoreExecutors.directExecutor());
1945 if (Config.REQUIRE_RTP_VERIFICATION) {
1946 return future;
1947 }
1948 return Futures.catching(
1949 future,
1950 CryptoFailedException.class,
1951 e -> {
1952 Log.w(
1953 Config.LOGTAG,
1954 id.account.getJid().asBareJid()
1955 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1956 e);
1957 return rtpContentMap;
1958 },
1959 MoreExecutors.directExecutor());
1960 } else {
1961 return Futures.immediateFuture(rtpContentMap);
1962 }
1963 }
1964
1965 private void sendSessionTerminate(final Reason reason) {
1966 sendSessionTerminate(reason, null);
1967 }
1968
1969 private void sendSessionTerminate(final Reason reason, final String text) {
1970 final State previous = this.state;
1971 final State target = reasonToState(reason);
1972 transitionOrThrow(target);
1973 if (previous != State.NULL) {
1974 writeLogMessage(target);
1975 }
1976 final JinglePacket jinglePacket =
1977 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1978 jinglePacket.setReason(reason, text);
1979 Log.d(Config.LOGTAG, jinglePacket.toString());
1980 send(jinglePacket);
1981 finish();
1982 }
1983
1984 private void sendTransportInfo(
1985 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1986 final RtpContentMap transportInfo;
1987 try {
1988 final RtpContentMap rtpContentMap =
1989 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1990 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1991 } catch (final Exception e) {
1992 Log.d(
1993 Config.LOGTAG,
1994 id.account.getJid().asBareJid()
1995 + ": unable to prepare transport-info from candidate for content="
1996 + contentName);
1997 return;
1998 }
1999 final JinglePacket jinglePacket =
2000 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2001 send(jinglePacket);
2002 }
2003
2004 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2005 // TODO send all candidates in one transport-info
2006 }
2007
2008
2009 private void send(final JinglePacket jinglePacket) {
2010 jinglePacket.setTo(id.with);
2011 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2012 }
2013
2014 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2015 if (response.getType() == IqPacket.TYPE.ERROR) {
2016 handleIqErrorResponse(response);
2017 return;
2018 }
2019 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2020 handleIqTimeoutResponse(response);
2021 }
2022 }
2023
2024 private void handleIqErrorResponse(final IqPacket response) {
2025 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2026 final String errorCondition = response.getErrorCondition();
2027 Log.d(
2028 Config.LOGTAG,
2029 id.account.getJid().asBareJid()
2030 + ": received IQ-error from "
2031 + response.getFrom()
2032 + " in RTP session. "
2033 + errorCondition);
2034 if (isTerminated()) {
2035 Log.i(
2036 Config.LOGTAG,
2037 id.account.getJid().asBareJid()
2038 + ": ignoring error because session was already terminated");
2039 return;
2040 }
2041 this.webRTCWrapper.close();
2042 final State target;
2043 if (Arrays.asList(
2044 "service-unavailable",
2045 "recipient-unavailable",
2046 "remote-server-not-found",
2047 "remote-server-timeout")
2048 .contains(errorCondition)) {
2049 target = State.TERMINATED_CONNECTIVITY_ERROR;
2050 } else {
2051 target = State.TERMINATED_APPLICATION_FAILURE;
2052 }
2053 transitionOrThrow(target);
2054 this.finish();
2055 }
2056
2057 private void handleIqTimeoutResponse(final IqPacket response) {
2058 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2059 Log.d(
2060 Config.LOGTAG,
2061 id.account.getJid().asBareJid()
2062 + ": received IQ timeout in RTP session with "
2063 + id.with
2064 + ". terminating with connectivity error");
2065 if (isTerminated()) {
2066 Log.i(
2067 Config.LOGTAG,
2068 id.account.getJid().asBareJid()
2069 + ": ignoring error because session was already terminated");
2070 return;
2071 }
2072 this.webRTCWrapper.close();
2073 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2074 this.finish();
2075 }
2076
2077 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2078 Log.d(
2079 Config.LOGTAG,
2080 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2081 this.webRTCWrapper.close();
2082 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2083 respondWithOutOfOrder(jinglePacket);
2084 this.finish();
2085 }
2086
2087 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2088 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2089 }
2090
2091 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2092 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2093 }
2094
2095 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2096 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2097 }
2098
2099 void respondWithJingleError(
2100 final IqPacket original,
2101 String jingleCondition,
2102 String condition,
2103 String conditionType) {
2104 jingleConnectionManager.respondWithJingleError(
2105 id.account, original, jingleCondition, condition, conditionType);
2106 }
2107
2108 private void respondOk(final JinglePacket jinglePacket) {
2109 xmppConnectionService.sendIqPacket(
2110 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2111 }
2112
2113 public RtpEndUserState getEndUserState() {
2114 switch (this.state) {
2115 case NULL:
2116 case PROPOSED:
2117 case SESSION_INITIALIZED:
2118 if (isInitiator()) {
2119 return RtpEndUserState.RINGING;
2120 } else {
2121 return RtpEndUserState.INCOMING_CALL;
2122 }
2123 case PROCEED:
2124 if (isInitiator()) {
2125 return RtpEndUserState.RINGING;
2126 } else {
2127 return RtpEndUserState.ACCEPTING_CALL;
2128 }
2129 case SESSION_INITIALIZED_PRE_APPROVED:
2130 if (isInitiator()) {
2131 return RtpEndUserState.RINGING;
2132 } else {
2133 return RtpEndUserState.CONNECTING;
2134 }
2135 case SESSION_ACCEPTED:
2136 final ContentAddition ca = getPendingContentAddition();
2137 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2138 return RtpEndUserState.INCOMING_CONTENT_ADD;
2139 }
2140 return getPeerConnectionStateAsEndUserState();
2141 case REJECTED:
2142 case REJECTED_RACED:
2143 case TERMINATED_DECLINED_OR_BUSY:
2144 if (isInitiator()) {
2145 return RtpEndUserState.DECLINED_OR_BUSY;
2146 } else {
2147 return RtpEndUserState.ENDED;
2148 }
2149 case TERMINATED_SUCCESS:
2150 case ACCEPTED:
2151 case RETRACTED:
2152 case TERMINATED_CANCEL_OR_TIMEOUT:
2153 return RtpEndUserState.ENDED;
2154 case RETRACTED_RACED:
2155 if (isInitiator()) {
2156 return RtpEndUserState.ENDED;
2157 } else {
2158 return RtpEndUserState.RETRACTED;
2159 }
2160 case TERMINATED_CONNECTIVITY_ERROR:
2161 return zeroDuration()
2162 ? RtpEndUserState.CONNECTIVITY_ERROR
2163 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2164 case TERMINATED_APPLICATION_FAILURE:
2165 return RtpEndUserState.APPLICATION_ERROR;
2166 case TERMINATED_SECURITY_ERROR:
2167 return RtpEndUserState.SECURITY_ERROR;
2168 }
2169 throw new IllegalStateException(
2170 String.format("%s has no equivalent EndUserState", this.state));
2171 }
2172
2173 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2174 final PeerConnection.PeerConnectionState state;
2175 try {
2176 state = webRTCWrapper.getState();
2177 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2178 // We usually close the WebRTCWrapper *before* transitioning so we might still
2179 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2180 return RtpEndUserState.ENDING_CALL;
2181 }
2182 switch (state) {
2183 case CONNECTED:
2184 return RtpEndUserState.CONNECTED;
2185 case NEW:
2186 case CONNECTING:
2187 return RtpEndUserState.CONNECTING;
2188 case CLOSED:
2189 return RtpEndUserState.ENDING_CALL;
2190 default:
2191 return zeroDuration()
2192 ? RtpEndUserState.CONNECTIVITY_ERROR
2193 : RtpEndUserState.RECONNECTING;
2194 }
2195 }
2196
2197 public ContentAddition getPendingContentAddition() {
2198 final RtpContentMap in = this.incomingContentAdd;
2199 final RtpContentMap out = this.outgoingContentAdd;
2200 if (out != null) {
2201 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2202 } else if (in != null) {
2203 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2204 } else {
2205 return null;
2206 }
2207 }
2208
2209 public Set<Media> getMedia() {
2210 final State current = getState();
2211 if (current == State.NULL) {
2212 if (isInitiator()) {
2213 return Preconditions.checkNotNull(
2214 this.proposedMedia, "RTP connection has not been initialized properly");
2215 }
2216 throw new IllegalStateException("RTP connection has not been initialized yet");
2217 }
2218 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2219 return Preconditions.checkNotNull(
2220 this.proposedMedia, "RTP connection has not been initialized properly");
2221 }
2222 final RtpContentMap localContentMap = getLocalContentMap();
2223 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2224 if (localContentMap != null) {
2225 return localContentMap.getMedia();
2226 } else if (initiatorContentMap != null) {
2227 return initiatorContentMap.getMedia();
2228 } else if (isTerminated()) {
2229 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2230 } else {
2231 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2232 }
2233 }
2234
2235 public boolean isVerified() {
2236 final String fingerprint = this.omemoVerification.getFingerprint();
2237 if (fingerprint == null) {
2238 return false;
2239 }
2240 final FingerprintStatus status =
2241 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2242 return status != null && status.isVerified();
2243 }
2244
2245 public boolean addMedia(final Media media) {
2246 final Set<Media> currentMedia = getMedia();
2247 if (currentMedia.contains(media)) {
2248 throw new IllegalStateException(String.format("%s has already been proposed", media));
2249 }
2250 // TODO add state protection - can only add while ACCEPTED or so
2251 Log.d(Config.LOGTAG,"adding media: "+media);
2252 return webRTCWrapper.addTrack(media);
2253 }
2254
2255 public synchronized void acceptCall() {
2256 switch (this.state) {
2257 case PROPOSED:
2258 cancelRingingTimeout();
2259 acceptCallFromProposed();
2260 break;
2261 case SESSION_INITIALIZED:
2262 cancelRingingTimeout();
2263 acceptCallFromSessionInitialized();
2264 break;
2265 case ACCEPTED:
2266 Log.w(
2267 Config.LOGTAG,
2268 id.account.getJid().asBareJid()
2269 + ": the call has already been accepted with another client. UI was just lagging behind");
2270 break;
2271 case PROCEED:
2272 case SESSION_ACCEPTED:
2273 Log.w(
2274 Config.LOGTAG,
2275 id.account.getJid().asBareJid()
2276 + ": the call has already been accepted. user probably double tapped the UI");
2277 break;
2278 default:
2279 throw new IllegalStateException("Can not accept call from " + this.state);
2280 }
2281 }
2282
2283 public void notifyPhoneCall() {
2284 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2285 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2286 rejectCall();
2287 } else {
2288 endCall();
2289 }
2290 }
2291
2292 public synchronized void rejectCall() {
2293 if (isTerminated()) {
2294 Log.w(
2295 Config.LOGTAG,
2296 id.account.getJid().asBareJid()
2297 + ": received rejectCall() when session has already been terminated. nothing to do");
2298 return;
2299 }
2300 switch (this.state) {
2301 case PROPOSED:
2302 rejectCallFromProposed();
2303 break;
2304 case SESSION_INITIALIZED:
2305 rejectCallFromSessionInitiate();
2306 break;
2307 default:
2308 throw new IllegalStateException("Can not reject call from " + this.state);
2309 }
2310 }
2311
2312 public synchronized void endCall() {
2313 if (isTerminated()) {
2314 Log.w(
2315 Config.LOGTAG,
2316 id.account.getJid().asBareJid()
2317 + ": received endCall() when session has already been terminated. nothing to do");
2318 return;
2319 }
2320 if (isInState(State.PROPOSED) && !isInitiator()) {
2321 rejectCallFromProposed();
2322 return;
2323 }
2324 if (isInState(State.PROCEED)) {
2325 if (isInitiator()) {
2326 retractFromProceed();
2327 } else {
2328 rejectCallFromProceed();
2329 }
2330 return;
2331 }
2332 if (isInitiator()
2333 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2334 this.webRTCWrapper.close();
2335 sendSessionTerminate(Reason.CANCEL);
2336 return;
2337 }
2338 if (isInState(State.SESSION_INITIALIZED)) {
2339 rejectCallFromSessionInitiate();
2340 return;
2341 }
2342 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2343 this.webRTCWrapper.close();
2344 sendSessionTerminate(Reason.SUCCESS);
2345 return;
2346 }
2347 if (isInState(
2348 State.TERMINATED_APPLICATION_FAILURE,
2349 State.TERMINATED_CONNECTIVITY_ERROR,
2350 State.TERMINATED_DECLINED_OR_BUSY)) {
2351 Log.d(
2352 Config.LOGTAG,
2353 "ignoring request to end call because already in state " + this.state);
2354 return;
2355 }
2356 throw new IllegalStateException(
2357 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2358 }
2359
2360 private void retractFromProceed() {
2361 Log.d(Config.LOGTAG, "retract from proceed");
2362 this.sendJingleMessage("retract");
2363 closeTransitionLogFinish(State.RETRACTED_RACED);
2364 }
2365
2366 private void closeTransitionLogFinish(final State state) {
2367 this.webRTCWrapper.close();
2368 transitionOrThrow(state);
2369 writeLogMessage(state);
2370 finish();
2371 }
2372
2373 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2374 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2375 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2376 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2377 }
2378
2379 private void acceptCallFromProposed() {
2380 transitionOrThrow(State.PROCEED);
2381 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2382 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2383 this.sendJingleMessage("proceed");
2384 }
2385
2386 private void rejectCallFromProposed() {
2387 transitionOrThrow(State.REJECTED);
2388 writeLogMessageMissed();
2389 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2390 this.sendJingleMessage("reject");
2391 finish();
2392 }
2393
2394 private void rejectCallFromProceed() {
2395 this.sendJingleMessage("reject");
2396 closeTransitionLogFinish(State.REJECTED_RACED);
2397 }
2398
2399 private void rejectCallFromSessionInitiate() {
2400 webRTCWrapper.close();
2401 sendSessionTerminate(Reason.DECLINE);
2402 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2403 }
2404
2405 private void sendJingleMessage(final String action) {
2406 sendJingleMessage(action, id.with);
2407 }
2408
2409 private void sendJingleMessage(final String action, final Jid to) {
2410 final MessagePacket messagePacket = new MessagePacket();
2411 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2412 messagePacket.setTo(to);
2413 final Element intent =
2414 messagePacket
2415 .addChild(action, Namespace.JINGLE_MESSAGE)
2416 .setAttribute("id", id.sessionId);
2417 if ("proceed".equals(action)) {
2418 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2419 if (isOmemoEnabled()) {
2420 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2421 final Element device =
2422 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2423 device.setAttribute("id", deviceId);
2424 }
2425 }
2426 messagePacket.addChild("store", "urn:xmpp:hints");
2427 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2428 }
2429
2430 private boolean isOmemoEnabled() {
2431 final Conversational conversational = message.getConversation();
2432 if (conversational instanceof Conversation) {
2433 return ((Conversation) conversational).getNextEncryption()
2434 == Message.ENCRYPTION_AXOLOTL;
2435 }
2436 return false;
2437 }
2438
2439 private void acceptCallFromSessionInitialized() {
2440 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2441 sendSessionAccept();
2442 }
2443
2444 private synchronized boolean isInState(State... state) {
2445 return Arrays.asList(state).contains(this.state);
2446 }
2447
2448 private boolean transition(final State target) {
2449 return transition(target, null);
2450 }
2451
2452 private synchronized boolean transition(final State target, final Runnable runnable) {
2453 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2454 if (validTransitions != null && validTransitions.contains(target)) {
2455 this.state = target;
2456 if (runnable != null) {
2457 runnable.run();
2458 }
2459 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2460 updateEndUserState();
2461 updateOngoingCallNotification();
2462 return true;
2463 } else {
2464 return false;
2465 }
2466 }
2467
2468 void transitionOrThrow(final State target) {
2469 if (!transition(target)) {
2470 throw new IllegalStateException(
2471 String.format("Unable to transition from %s to %s", this.state, target));
2472 }
2473 }
2474
2475 @Override
2476 public void onIceCandidate(final IceCandidate iceCandidate) {
2477 final RtpContentMap rtpContentMap =
2478 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2479 final IceUdpTransportInfo.Credentials credentials;
2480 try {
2481 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2482 } catch (final IllegalArgumentException e) {
2483 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2484 return;
2485 }
2486 final String uFrag = credentials.ufrag;
2487 final IceUdpTransportInfo.Candidate candidate =
2488 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2489 if (candidate == null) {
2490 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2491 return;
2492 }
2493 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2494 sendTransportInfo(iceCandidate.sdpMid, candidate);
2495 }
2496
2497 @Override
2498 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2499 Log.d(
2500 Config.LOGTAG,
2501 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2502 this.stateHistory.add(newState);
2503 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2504 this.sessionDuration.start();
2505 updateOngoingCallNotification();
2506 } else if (this.sessionDuration.isRunning()) {
2507 this.sessionDuration.stop();
2508 updateOngoingCallNotification();
2509 }
2510
2511 final boolean neverConnected =
2512 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2513
2514 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2515 if (neverConnected) {
2516 if (isTerminated()) {
2517 Log.d(
2518 Config.LOGTAG,
2519 id.account.getJid().asBareJid()
2520 + ": not sending session-terminate after connectivity error because session is already in state "
2521 + this.state);
2522 return;
2523 }
2524 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2525 return;
2526 } else {
2527 this.restartIce();
2528 }
2529 }
2530 updateEndUserState();
2531 }
2532
2533 private void restartIce() {
2534 this.stateHistory.clear();
2535 this.webRTCWrapper.restartIceAsync();
2536 }
2537
2538 @Override
2539 public void onRenegotiationNeeded() {
2540 this.webRTCWrapper.execute(this::renegotiate);
2541 }
2542
2543 private void renegotiate() {
2544 final SessionDescription sessionDescription;
2545 try {
2546 sessionDescription = setLocalSessionDescription();
2547 } catch (final Exception e) {
2548 final Throwable cause = Throwables.getRootCause(e);
2549 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2550 webRTCWrapper.close();
2551 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2552 return;
2553 }
2554 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2555 final RtpContentMap currentContentMap = getLocalContentMap();
2556 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2557 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2558
2559 Log.d(
2560 Config.LOGTAG,
2561 id.getAccount().getJid().asBareJid()
2562 + ": renegotiate. iceRestart="
2563 + iceRestart
2564 + " content id diff="
2565 + diff);
2566
2567 if (diff.hasModifications() && iceRestart) {
2568 webRTCWrapper.close();
2569 sendSessionTerminate(
2570 Reason.FAILED_APPLICATION,
2571 "WebRTC unexpectedly tried to modify content and transport at once");
2572 return;
2573 }
2574
2575 if (iceRestart) {
2576 initiateIceRestart(rtpContentMap);
2577 return;
2578 } else if (diff.isEmpty()) {
2579 Log.d(
2580 Config.LOGTAG,
2581 "renegotiation. nothing to do. SignalingState="
2582 + this.webRTCWrapper.getSignalingState());
2583 }
2584
2585 if (diff.added.size() > 0) {
2586 modifyLocalContentMap(rtpContentMap);
2587 sendContentAdd(rtpContentMap, diff.added);
2588 }
2589 }
2590
2591 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2592 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2593 final JinglePacket jinglePacket =
2594 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2595 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2596 jinglePacket.setTo(id.with);
2597 xmppConnectionService.sendIqPacket(
2598 id.account,
2599 jinglePacket,
2600 (account, response) -> {
2601 if (response.getType() == IqPacket.TYPE.RESULT) {
2602 Log.d(Config.LOGTAG, "received success to our ice restart");
2603 setLocalContentMap(rtpContentMap);
2604 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2605 return;
2606 }
2607 if (response.getType() == IqPacket.TYPE.ERROR) {
2608 if (isTieBreak(response)) {
2609 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2610 return;
2611 }
2612 handleIqErrorResponse(response);
2613 }
2614 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2615 handleIqTimeoutResponse(response);
2616 }
2617 });
2618 }
2619
2620 private boolean isTieBreak(final IqPacket response) {
2621 final Element error = response.findChild("error");
2622 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2623 }
2624
2625 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2626 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2627 this.outgoingContentAdd = contentAdd;
2628 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2629 prepareOutgoingContentMap(contentAdd);
2630 Futures.addCallback(
2631 outgoingContentMapFuture,
2632 new FutureCallback<RtpContentMap>() {
2633 @Override
2634 public void onSuccess(final RtpContentMap outgoingContentMap) {
2635 sendContentAdd(outgoingContentMap);
2636 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2637 }
2638
2639 @Override
2640 public void onFailure(@NonNull Throwable throwable) {
2641 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2642 }
2643 },
2644 MoreExecutors.directExecutor());
2645 }
2646
2647 private void sendContentAdd(final RtpContentMap contentAdd) {
2648
2649 final JinglePacket jinglePacket =
2650 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2651 jinglePacket.setTo(id.with);
2652 xmppConnectionService.sendIqPacket(
2653 id.account,
2654 jinglePacket,
2655 (connection, response) -> {
2656 if (response.getType() == IqPacket.TYPE.RESULT) {
2657 Log.d(
2658 Config.LOGTAG,
2659 id.getAccount().getJid().asBareJid()
2660 + ": received ACK to our content-add");
2661 return;
2662 }
2663 if (response.getType() == IqPacket.TYPE.ERROR) {
2664 if (isTieBreak(response)) {
2665 this.outgoingContentAdd = null;
2666 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2667 return;
2668 }
2669 handleIqErrorResponse(response);
2670 }
2671 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2672 handleIqTimeoutResponse(response);
2673 }
2674 });
2675 }
2676
2677 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2678 if (isInitiator()) {
2679 this.initiatorRtpContentMap = rtpContentMap;
2680 } else {
2681 this.responderRtpContentMap = rtpContentMap;
2682 }
2683 }
2684
2685 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2686 if (isInitiator()) {
2687 this.responderRtpContentMap = rtpContentMap;
2688 } else {
2689 this.initiatorRtpContentMap = rtpContentMap;
2690 }
2691 }
2692
2693 // this method is to be used for content map modifications that modify media
2694 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2695 final RtpContentMap activeContents = rtpContentMap.activeContents();
2696 setLocalContentMap(activeContents);
2697 this.webRTCWrapper.switchSpeakerPhonePreference(
2698 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2699 updateEndUserState();
2700 }
2701
2702 private SessionDescription setLocalSessionDescription()
2703 throws ExecutionException, InterruptedException {
2704 final org.webrtc.SessionDescription sessionDescription =
2705 this.webRTCWrapper.setLocalDescription().get();
2706 return SessionDescription.parse(sessionDescription.description);
2707 }
2708
2709 private void closeWebRTCSessionAfterFailedConnection() {
2710 this.webRTCWrapper.close();
2711 synchronized (this) {
2712 if (isTerminated()) {
2713 Log.d(
2714 Config.LOGTAG,
2715 id.account.getJid().asBareJid()
2716 + ": no need to send session-terminate after failed connection. Other party already did");
2717 return;
2718 }
2719 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2720 }
2721 }
2722
2723 public boolean zeroDuration() {
2724 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2725 }
2726
2727 public long getCallDuration() {
2728 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2729 }
2730
2731 public AppRTCAudioManager getAudioManager() {
2732 return webRTCWrapper.getAudioManager();
2733 }
2734
2735 public boolean isMicrophoneEnabled() {
2736 return webRTCWrapper.isMicrophoneEnabled();
2737 }
2738
2739 public boolean setMicrophoneEnabled(final boolean enabled) {
2740 return webRTCWrapper.setMicrophoneEnabled(enabled);
2741 }
2742
2743 public boolean isVideoEnabled() {
2744 return webRTCWrapper.isVideoEnabled();
2745 }
2746
2747 public void setVideoEnabled(final boolean enabled) {
2748 webRTCWrapper.setVideoEnabled(enabled);
2749 }
2750
2751 public boolean isCameraSwitchable() {
2752 return webRTCWrapper.isCameraSwitchable();
2753 }
2754
2755 public boolean isFrontCamera() {
2756 return webRTCWrapper.isFrontCamera();
2757 }
2758
2759 public ListenableFuture<Boolean> switchCamera() {
2760 return webRTCWrapper.switchCamera();
2761 }
2762
2763 @Override
2764 public void onAudioDeviceChanged(
2765 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2766 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2767 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2768 selectedAudioDevice, availableAudioDevices);
2769 }
2770
2771 private void updateEndUserState() {
2772 final RtpEndUserState endUserState = getEndUserState();
2773 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2774 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2775 id.account, id.with, id.sessionId, endUserState);
2776 }
2777
2778 private void updateOngoingCallNotification() {
2779 final State state = this.state;
2780 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2781 final boolean reconnecting;
2782 if (state == State.SESSION_ACCEPTED) {
2783 reconnecting =
2784 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2785 } else {
2786 reconnecting = false;
2787 }
2788 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2789 } else {
2790 xmppConnectionService.removeOngoingCall();
2791 }
2792 }
2793
2794 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2795 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2796 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2797 request.setTo(id.account.getDomain());
2798 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2799 xmppConnectionService.sendIqPacket(
2800 id.account,
2801 request,
2802 (account, response) -> {
2803 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2804 new ImmutableList.Builder<>();
2805 if (response.getType() == IqPacket.TYPE.RESULT) {
2806 final Element services =
2807 response.findChild(
2808 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2809 final List<Element> children =
2810 services == null
2811 ? Collections.emptyList()
2812 : services.getChildren();
2813 for (final Element child : children) {
2814 if ("service".equals(child.getName())) {
2815 final String type = child.getAttribute("type");
2816 final String host = child.getAttribute("host");
2817 final String sport = child.getAttribute("port");
2818 final Integer port =
2819 sport == null ? null : Ints.tryParse(sport);
2820 final String transport = child.getAttribute("transport");
2821 final String username = child.getAttribute("username");
2822 final String password = child.getAttribute("password");
2823 if (Strings.isNullOrEmpty(host) || port == null) {
2824 continue;
2825 }
2826 if (port < 0 || port > 65535) {
2827 continue;
2828 }
2829
2830
2831
2832
2833 if (Arrays.asList("stun", "stuns", "turn", "turns")
2834 .contains(type)
2835 && Arrays.asList("udp", "tcp").contains(transport)) {
2836 if (Arrays.asList("stuns", "turns").contains(type)
2837 && "udp".equals(transport)) {
2838 Log.d(
2839 Config.LOGTAG,
2840 id.account.getJid().asBareJid()
2841 + ": skipping invalid combination of udp/tls in external services");
2842 continue;
2843 }
2844
2845 // STUN URLs do not support a query section since M110
2846 final String uri;
2847 if (Arrays.asList("stun","stuns").contains(type)) {
2848 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2849 } else {
2850 uri = String.format(
2851 "%s:%s:%s?transport=%s",
2852 type,
2853 IP.wrapIPv6(host),
2854 port,
2855 transport);
2856 }
2857
2858 final PeerConnection.IceServer.Builder iceServerBuilder =
2859 PeerConnection.IceServer.builder(uri);
2860 iceServerBuilder.setTlsCertPolicy(
2861 PeerConnection.TlsCertPolicy
2862 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2863 if (username != null && password != null) {
2864 iceServerBuilder.setUsername(username);
2865 iceServerBuilder.setPassword(password);
2866 } else if (Arrays.asList("turn", "turns").contains(type)) {
2867 // The WebRTC spec requires throwing an
2868 // InvalidAccessError when username (from libwebrtc
2869 // source coder)
2870 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2871 Log.d(
2872 Config.LOGTAG,
2873 id.account.getJid().asBareJid()
2874 + ": skipping "
2875 + type
2876 + "/"
2877 + transport
2878 + " without username and password");
2879 continue;
2880 }
2881 final PeerConnection.IceServer iceServer =
2882 iceServerBuilder.createIceServer();
2883 Log.d(
2884 Config.LOGTAG,
2885 id.account.getJid().asBareJid()
2886 + ": discovered ICE Server: "
2887 + iceServer);
2888 listBuilder.add(iceServer);
2889 }
2890 }
2891 }
2892 }
2893 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2894 if (iceServers.size() == 0) {
2895 Log.w(
2896 Config.LOGTAG,
2897 id.account.getJid().asBareJid()
2898 + ": no ICE server found "
2899 + response);
2900 }
2901 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2902 });
2903 } else {
2904 Log.w(
2905 Config.LOGTAG,
2906 id.account.getJid().asBareJid() + ": has no external service discovery");
2907 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2908 }
2909 }
2910
2911 private void finish() {
2912 if (isTerminated()) {
2913 this.cancelRingingTimeout();
2914 this.webRTCWrapper.verifyClosed();
2915 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2916 this.jingleConnectionManager.finishConnectionOrThrow(this);
2917 } else {
2918 throw new IllegalStateException(
2919 String.format("Unable to call finish from %s", this.state));
2920 }
2921 }
2922
2923 private void writeLogMessage(final State state) {
2924 final long duration = getCallDuration();
2925 if (state == State.TERMINATED_SUCCESS
2926 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2927 writeLogMessageSuccess(duration);
2928 } else {
2929 writeLogMessageMissed();
2930 }
2931 }
2932
2933 private void writeLogMessageSuccess(final long duration) {
2934 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2935 this.writeMessage();
2936 }
2937
2938 private void writeLogMessageMissed() {
2939 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2940 this.writeMessage();
2941 }
2942
2943 private void writeMessage() {
2944 final Conversational conversational = message.getConversation();
2945 if (conversational instanceof Conversation) {
2946 ((Conversation) conversational).add(this.message);
2947 xmppConnectionService.createMessageAsync(message);
2948 xmppConnectionService.updateConversationUi();
2949 } else {
2950 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2951 }
2952 }
2953
2954 public State getState() {
2955 return this.state;
2956 }
2957
2958 boolean isTerminated() {
2959 return TERMINATED.contains(this.state);
2960 }
2961
2962 public Optional<VideoTrack> getLocalVideoTrack() {
2963 return webRTCWrapper.getLocalVideoTrack();
2964 }
2965
2966 public Optional<VideoTrack> getRemoteVideoTrack() {
2967 return webRTCWrapper.getRemoteVideoTrack();
2968 }
2969
2970 public EglBase.Context getEglBaseContext() {
2971 return webRTCWrapper.getEglBaseContext();
2972 }
2973
2974 void setProposedMedia(final Set<Media> media) {
2975 this.proposedMedia = media;
2976 }
2977
2978 public void fireStateUpdate() {
2979 final RtpEndUserState endUserState = getEndUserState();
2980 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2981 id.account, id.with, id.sessionId, endUserState);
2982 }
2983
2984 public boolean isSwitchToVideoAvailable() {
2985 final boolean prerequisite =
2986 Media.audioOnly(getMedia())
2987 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2988 .contains(getEndUserState());
2989 return prerequisite && remoteHasVideoFeature();
2990 }
2991
2992 private boolean remoteHasVideoFeature() {
2993 final Contact contact = id.getContact();
2994 final Presence presence =
2995 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2996 final ServiceDiscoveryResult serviceDiscoveryResult =
2997 presence == null ? null : presence.getServiceDiscoveryResult();
2998 final List<String> features =
2999 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
3000 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
3001 }
3002
3003 private interface OnIceServersDiscovered {
3004 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3005 }
3006}