1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50
51import im.conversations.android.xmpp.model.jingle.Jingle;
52import im.conversations.android.xmpp.model.stanza.Iq;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROPOSED,
78 State.PROCEED,
79 State.SESSION_INITIALIZED_PRE_APPROVED,
80 State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82
83 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
84 private final Queue<
85 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
86 pendingIceCandidates = new LinkedList<>();
87 private final OmemoVerification omemoVerification = new OmemoVerification();
88 private final CallIntegration callIntegration;
89 private final Message message;
90
91 private Set<Media> proposedMedia;
92 private RtpContentMap initiatorRtpContentMap;
93 private RtpContentMap responderRtpContentMap;
94 private RtpContentMap incomingContentAdd;
95 private RtpContentMap outgoingContentAdd;
96 private IceUdpTransportInfo.Setup peerDtlsSetup;
97 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
98 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
99 private ScheduledFuture<?> ringingTimeoutFuture;
100
101 JingleRtpConnection(
102 final JingleConnectionManager jingleConnectionManager,
103 final Id id,
104 final Jid initiator) {
105 this(
106 jingleConnectionManager,
107 id,
108 initiator,
109 new CallIntegration(
110 jingleConnectionManager
111 .getXmppConnectionService()
112 .getApplicationContext()));
113 this.callIntegration.setAddress(
114 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
115 final var contact = id.getContact();
116 this.callIntegration.setCallerDisplayName(
117 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
118 this.callIntegration.setInitialized();
119 }
120
121 JingleRtpConnection(
122 final JingleConnectionManager jingleConnectionManager,
123 final Id id,
124 final Jid initiator,
125 final CallIntegration callIntegration) {
126 super(jingleConnectionManager, id, initiator);
127 final Conversation conversation =
128 jingleConnectionManager
129 .getXmppConnectionService()
130 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
131 this.message =
132 new Message(
133 conversation,
134 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
135 Message.TYPE_RTP_SESSION,
136 id.sessionId);
137 this.callIntegration = callIntegration;
138 this.callIntegration.setCallback(this);
139 }
140
141 @Override
142 synchronized void deliverPacket(final Iq iq) {
143 final var jingle = iq.getExtension(Jingle.class);
144 switch (jingle.getAction()) {
145 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
146 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
147 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
148 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
149 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
150 case CONTENT_ACCEPT -> receiveContentAccept(iq);
151 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
152 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
153 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
154 default -> {
155 respondOk(iq);
156 Log.d(
157 Config.LOGTAG,
158 String.format(
159 "%s: received unhandled jingle action %s",
160 id.account.getJid().asBareJid(), jingle.getAction()));
161 }
162 }
163 }
164
165 @Override
166 synchronized void notifyRebound() {
167 if (isTerminated()) {
168 return;
169 }
170 webRTCWrapper.close();
171 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
172 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
173 }
174 if (isInState(
175 State.SESSION_INITIALIZED,
176 State.SESSION_INITIALIZED_PRE_APPROVED,
177 State.SESSION_ACCEPTED)) {
178 // we might have already changed resources (full jid) at this point; so this might not
179 // even reach the other party
180 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
181 } else {
182 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
183 finish();
184 }
185 }
186
187 private void receiveSessionTerminate(final Iq jinglePacket) {
188 respondOk(jinglePacket);
189 final var jingle = jinglePacket.getExtension(Jingle.class);
190 final Jingle.ReasonWrapper wrapper = jingle.getReason();
191 final State previous = this.state;
192 Log.d(
193 Config.LOGTAG,
194 id.account.getJid().asBareJid()
195 + ": received session terminate reason="
196 + wrapper.reason
197 + "("
198 + Strings.nullToEmpty(wrapper.text)
199 + ") while in state "
200 + previous);
201 if (TERMINATED.contains(previous)) {
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": ignoring session terminate because already in "
206 + previous);
207 return;
208 }
209 webRTCWrapper.close();
210 final State target = reasonToState(wrapper.reason);
211 transitionOrThrow(target);
212 writeLogMessage(target);
213 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
214 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
215 }
216 finish();
217 }
218
219 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
220 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
221 // INITIALIZED only after transport-info has been received
222 if (isInState(
223 State.NULL,
224 State.PROCEED,
225 State.SESSION_INITIALIZED,
226 State.SESSION_INITIALIZED_PRE_APPROVED,
227 State.SESSION_ACCEPTED)) {
228 final RtpContentMap contentMap;
229 try {
230 contentMap = RtpContentMap.of(jingle);
231 } catch (final IllegalArgumentException | NullPointerException e) {
232 Log.d(
233 Config.LOGTAG,
234 id.account.getJid().asBareJid()
235 + ": improperly formatted contents; ignoring",
236 e);
237 respondOk(jinglePacket);
238 return;
239 }
240 receiveTransportInfo(jinglePacket, contentMap);
241 } else {
242 if (isTerminated()) {
243 respondOk(jinglePacket);
244 Log.d(
245 Config.LOGTAG,
246 id.account.getJid().asBareJid()
247 + ": ignoring out-of-order transport info; we where already terminated");
248 } else {
249 Log.d(
250 Config.LOGTAG,
251 id.account.getJid().asBareJid()
252 + ": received transport info while in state="
253 + this.state);
254 terminateWithOutOfOrder(jinglePacket);
255 }
256 }
257 }
258
259 private void receiveTransportInfo(
260 final Iq jinglePacket, final RtpContentMap contentMap) {
261 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
262 candidates = contentMap.contents.entrySet();
263 final RtpContentMap remote = getRemoteContentMap();
264 final Set<String> remoteContentIds =
265 remote == null ? Collections.emptySet() : remote.contents.keySet();
266 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
267 Log.d(
268 Config.LOGTAG,
269 "received transport-info for unknown contents "
270 + contentMap.contents.keySet()
271 + " (known: "
272 + remoteContentIds
273 + ")");
274 respondOk(jinglePacket);
275 pendingIceCandidates.addAll(candidates);
276 return;
277 }
278 if (this.state != State.SESSION_ACCEPTED) {
279 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
280 respondOk(jinglePacket);
281 pendingIceCandidates.addAll(candidates);
282 return;
283 }
284 // zero candidates + modified credentials are an ICE restart offer
285 if (checkForIceRestart(jinglePacket, contentMap)) {
286 return;
287 }
288 respondOk(jinglePacket);
289 try {
290 processCandidates(candidates);
291 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
292 Log.w(
293 Config.LOGTAG,
294 id.account.getJid().asBareJid()
295 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
296 }
297 }
298
299 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
300 final RtpContentMap modification;
301 try {
302 modification = RtpContentMap.of(jingle);
303 modification.requireContentDescriptions();
304 } catch (final RuntimeException e) {
305 Log.d(
306 Config.LOGTAG,
307 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
308 Throwables.getRootCause(e));
309 respondOk(iq);
310 webRTCWrapper.close();
311 sendSessionTerminate(Reason.of(e), e.getMessage());
312 return;
313 }
314 if (isInState(State.SESSION_ACCEPTED)) {
315 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
316 final ListenableFuture<RtpContentMap> future =
317 receiveRtpContentMap(
318 modification,
319 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
320 Futures.addCallback(
321 future,
322 new FutureCallback<>() {
323 @Override
324 public void onSuccess(final RtpContentMap rtpContentMap) {
325 receiveContentAdd(iq, rtpContentMap);
326 }
327
328 @Override
329 public void onFailure(@NonNull Throwable throwable) {
330 respondOk(iq);
331 final Throwable rootCause = Throwables.getRootCause(throwable);
332 Log.d(
333 Config.LOGTAG,
334 id.account.getJid().asBareJid()
335 + ": improperly formatted contents in content-add",
336 throwable);
337 webRTCWrapper.close();
338 sendSessionTerminate(
339 Reason.ofThrowable(rootCause), rootCause.getMessage());
340 }
341 },
342 MoreExecutors.directExecutor());
343 } else {
344 terminateWithOutOfOrder(iq);
345 }
346 }
347
348 private void receiveContentAdd(
349 final Iq jinglePacket, final RtpContentMap modification) {
350 final RtpContentMap remote = getRemoteContentMap();
351 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
352 respondOk(jinglePacket);
353 this.webRTCWrapper.close();
354 sendSessionTerminate(
355 Reason.FAILED_APPLICATION,
356 String.format(
357 "contents with names %s already exists",
358 Joiner.on(", ").join(modification.getNames())));
359 return;
360 }
361 final ContentAddition contentAddition =
362 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
363
364 final RtpContentMap outgoing = this.outgoingContentAdd;
365 final Set<ContentAddition.Summary> outgoingContentAddSummary =
366 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
367
368 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
369 if (isInitiator()) {
370 Log.d(
371 Config.LOGTAG,
372 id.getAccount().getJid().asBareJid()
373 + ": respond with tie break to matching content-add offer");
374 respondWithTieBreak(jinglePacket);
375 } else {
376 Log.d(
377 Config.LOGTAG,
378 id.getAccount().getJid().asBareJid()
379 + ": automatically accept matching content-add offer");
380 acceptContentAdd(contentAddition.summary, modification);
381 }
382 return;
383 }
384
385 // once we can display multiple video tracks we can be more loose with this condition
386 // theoretically it should also be fine to automatically accept audio only contents
387 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
388 Log.d(
389 Config.LOGTAG,
390 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
391 this.incomingContentAdd = modification;
392 respondOk(jinglePacket);
393 updateEndUserState();
394 } else {
395 respondOk(jinglePacket);
396 // TODO do we want to add a reason?
397 rejectContentAdd(modification);
398 }
399 }
400
401 private void receiveContentAccept(final Iq jinglePacket) {
402 final var jingle = jinglePacket.getExtension(Jingle.class);
403 final RtpContentMap receivedContentAccept;
404 try {
405 receivedContentAccept = RtpContentMap.of(jingle);
406 receivedContentAccept.requireContentDescriptions();
407 } catch (final RuntimeException e) {
408 Log.d(
409 Config.LOGTAG,
410 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
411 Throwables.getRootCause(e));
412 respondOk(jinglePacket);
413 webRTCWrapper.close();
414 sendSessionTerminate(Reason.of(e), e.getMessage());
415 return;
416 }
417
418 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
419 if (outgoingContentAdd == null) {
420 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
421 terminateWithOutOfOrder(jinglePacket);
422 return;
423 }
424 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
425 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
426 this.outgoingContentAdd = null;
427 respondOk(jinglePacket);
428 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
429 final ListenableFuture<RtpContentMap> future =
430 receiveRtpContentMap(
431 receivedContentAccept,
432 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
433 Futures.addCallback(
434 future,
435 new FutureCallback<>() {
436 @Override
437 public void onSuccess(final RtpContentMap result) {
438 receiveContentAccept(result);
439 }
440
441 @Override
442 public void onFailure(@NonNull final Throwable throwable) {
443 webRTCWrapper.close();
444 sendSessionTerminate(
445 Reason.ofThrowable(throwable), throwable.getMessage());
446 }
447 },
448 MoreExecutors.directExecutor());
449 } else {
450 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
451 terminateWithOutOfOrder(jinglePacket);
452 }
453 }
454
455 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
456 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
457 final RtpContentMap modifiedContentMap =
458 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
459
460 setRemoteContentMap(modifiedContentMap);
461
462 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
463
464 final org.webrtc.SessionDescription sdp =
465 new org.webrtc.SessionDescription(
466 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
467
468 try {
469 this.webRTCWrapper.setRemoteDescription(sdp).get();
470 } catch (final Exception e) {
471 final Throwable cause = Throwables.getRootCause(e);
472 Log.d(
473 Config.LOGTAG,
474 id.getAccount().getJid().asBareJid()
475 + ": unable to set remote description after receiving content-accept",
476 cause);
477 webRTCWrapper.close();
478 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
479 return;
480 }
481 Log.d(
482 Config.LOGTAG,
483 id.getAccount().getJid().asBareJid()
484 + ": remote has accepted content-add "
485 + ContentAddition.summary(receivedContentAccept));
486 processCandidates(receivedContentAccept.contents.entrySet());
487 updateEndUserState();
488 }
489
490 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
491 if (this.state != State.SESSION_ACCEPTED) {
492 terminateWithOutOfOrder(jinglePacket);
493 return;
494 }
495 final Map<String, Content.Senders> modification =
496 Maps.transformEntries(
497 jingle.getJingleContents(), (key, value) -> value.getSenders());
498 final boolean isInitiator = isInitiator();
499 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
500 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
501 final Set<String> currentOutgoingMediaIds =
502 currentOutgoing == null
503 ? Collections.emptySet()
504 : currentOutgoing.contents.keySet();
505 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
506 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
507 respondOk(jinglePacket);
508 final RtpContentMap modifiedContentMap;
509 try {
510 modifiedContentMap =
511 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
512 } catch (final IllegalArgumentException e) {
513 webRTCWrapper.close();
514 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
515 return;
516 }
517 this.outgoingContentAdd = modifiedContentMap;
518 Log.d(
519 Config.LOGTAG,
520 id.account.getJid().asBareJid()
521 + ": processed content-modification for pending content-add");
522 } else if (remoteContentMap != null
523 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
524 respondOk(jinglePacket);
525 final RtpContentMap modifiedRemoteContentMap;
526 try {
527 modifiedRemoteContentMap =
528 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
529 } catch (final IllegalArgumentException e) {
530 webRTCWrapper.close();
531 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
532 return;
533 }
534 final SessionDescription offer;
535 try {
536 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
537 } catch (final IllegalArgumentException | NullPointerException e) {
538 Log.d(
539 Config.LOGTAG,
540 id.getAccount().getJid().asBareJid()
541 + ": unable convert offer from content-modify to SDP",
542 e);
543 webRTCWrapper.close();
544 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
545 return;
546 }
547 Log.d(
548 Config.LOGTAG,
549 id.account.getJid().asBareJid() + ": auto accepting content-modification");
550 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
551 } else {
552 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
553 respondWithItemNotFound(jinglePacket);
554 }
555 }
556
557 private void autoAcceptContentModify(
558 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
559 this.setRemoteContentMap(modifiedRemoteContentMap);
560 final org.webrtc.SessionDescription sdp =
561 new org.webrtc.SessionDescription(
562 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
563 try {
564 this.webRTCWrapper.setRemoteDescription(sdp).get();
565 // auto accept is only done when we already have tracks
566 final SessionDescription answer = setLocalSessionDescription();
567 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
568 modifyLocalContentMap(rtpContentMap);
569 // we do not need to send an answer but do we have to resend the candidates currently in
570 // SDP?
571 // resendCandidatesFromSdp(answer);
572 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
573 } catch (final Exception e) {
574 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
575 webRTCWrapper.close();
576 sendSessionTerminate(Reason.FAILED_APPLICATION);
577 }
578 }
579
580 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
581 final SessionDescription answer) {
582 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
583 new ImmutableMultimap.Builder<>();
584 for (final SessionDescription.Media media : answer.media) {
585 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
586 if (Strings.isNullOrEmpty(mid)) {
587 continue;
588 }
589 for (final String sdpCandidate : media.attributes.get("candidate")) {
590 final IceUdpTransportInfo.Candidate candidate =
591 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
592 if (candidate != null) {
593 candidateBuilder.put(mid, candidate);
594 }
595 }
596 }
597 return candidateBuilder.build();
598 }
599
600 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
601 final RtpContentMap receivedContentReject;
602 try {
603 receivedContentReject = RtpContentMap.of(jingle);
604 } catch (final RuntimeException e) {
605 Log.d(
606 Config.LOGTAG,
607 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
608 Throwables.getRootCause(e));
609 respondOk(jinglePacket);
610 this.webRTCWrapper.close();
611 sendSessionTerminate(Reason.of(e), e.getMessage());
612 return;
613 }
614
615 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
616 if (outgoingContentAdd == null) {
617 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
618 terminateWithOutOfOrder(jinglePacket);
619 return;
620 }
621 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
622 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
623 this.outgoingContentAdd = null;
624 respondOk(jinglePacket);
625 Log.d(Config.LOGTAG, jinglePacket.toString());
626 receiveContentReject(ourSummary);
627 } else {
628 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
629 terminateWithOutOfOrder(jinglePacket);
630 }
631 }
632
633 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
634 try {
635 this.webRTCWrapper.removeTrack(Media.VIDEO);
636 final RtpContentMap localContentMap = customRollback();
637 modifyLocalContentMap(localContentMap);
638 } catch (final Exception e) {
639 final Throwable cause = Throwables.getRootCause(e);
640 Log.d(
641 Config.LOGTAG,
642 id.getAccount().getJid().asBareJid()
643 + ": unable to rollback local description after receiving content-reject",
644 cause);
645 webRTCWrapper.close();
646 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
647 return;
648 }
649 Log.d(
650 Config.LOGTAG,
651 id.getAccount().getJid().asBareJid()
652 + ": remote has rejected our content-add "
653 + summary);
654 }
655
656 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
657 final RtpContentMap receivedContentRemove;
658 try {
659 receivedContentRemove = RtpContentMap.of(jingle);
660 receivedContentRemove.requireContentDescriptions();
661 } catch (final RuntimeException e) {
662 Log.d(
663 Config.LOGTAG,
664 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
665 Throwables.getRootCause(e));
666 respondOk(jinglePacket);
667 this.webRTCWrapper.close();
668 sendSessionTerminate(Reason.of(e), e.getMessage());
669 return;
670 }
671 respondOk(jinglePacket);
672 receiveContentRemove(receivedContentRemove);
673 }
674
675 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
676 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
677 final Set<ContentAddition.Summary> contentAddSummary =
678 incomingContentAdd == null
679 ? Collections.emptySet()
680 : ContentAddition.summary(incomingContentAdd);
681 final Set<ContentAddition.Summary> removeSummary =
682 ContentAddition.summary(receivedContentRemove);
683 if (contentAddSummary.equals(removeSummary)) {
684 this.incomingContentAdd = null;
685 updateEndUserState();
686 } else {
687 webRTCWrapper.close();
688 sendSessionTerminate(
689 Reason.FAILED_APPLICATION,
690 String.format(
691 "%s only supports %s as a means to retract a not yet accepted %s",
692 BuildConfig.APP_NAME,
693 Jingle.Action.CONTENT_REMOVE,
694 Jingle.Action.CONTENT_ADD));
695 }
696 }
697
698 public synchronized void retractContentAdd() {
699 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
700 if (outgoingContentAdd == null) {
701 throw new IllegalStateException("Not outgoing content add");
702 }
703 try {
704 webRTCWrapper.removeTrack(Media.VIDEO);
705 final RtpContentMap localContentMap = customRollback();
706 modifyLocalContentMap(localContentMap);
707 } catch (final Exception e) {
708 final Throwable cause = Throwables.getRootCause(e);
709 Log.d(
710 Config.LOGTAG,
711 id.getAccount().getJid().asBareJid()
712 + ": unable to rollback local description after trying to retract content-add",
713 cause);
714 webRTCWrapper.close();
715 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
716 return;
717 }
718 this.outgoingContentAdd = null;
719 final Iq retract =
720 outgoingContentAdd
721 .toStub()
722 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
723 this.send(retract);
724 Log.d(
725 Config.LOGTAG,
726 id.getAccount().getJid()
727 + ": retract content-add "
728 + ContentAddition.summary(outgoingContentAdd));
729 }
730
731 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
732 final SessionDescription sdp = setLocalSessionDescription();
733 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
734 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
735 this.webRTCWrapper
736 .setRemoteDescription(
737 new org.webrtc.SessionDescription(
738 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
739 .get();
740 return localRtpContentMap;
741 }
742
743 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
744 final RtpContentMap currentRemote = getRemoteContentMap();
745 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
746 if (diff.isEmpty()) {
747 throw new IllegalStateException(
748 "Unexpected rollback condition. No difference between local and remote");
749 }
750 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
751 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
752 final RtpContentMap nextRemote =
753 currentRemote.addContent(
754 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
755 return SessionDescription.of(nextRemote, isResponder());
756 }
757 throw new IllegalStateException(
758 "Unexpected rollback condition. Senders were not uniformly none");
759 }
760
761 public synchronized void acceptContentAdd(
762 @NonNull final Set<ContentAddition.Summary> contentAddition) {
763 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
764 if (incomingContentAdd == null) {
765 throw new IllegalStateException("No incoming content add");
766 }
767
768 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
769 this.incomingContentAdd = null;
770 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
771 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
772 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
773 Log.d(
774 Config.LOGTAG,
775 "content addition is receive only. we want to upgrade to 'both'");
776 final RtpContentMap modifiedSenders =
777 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
778 final Iq proposedContentModification =
779 modifiedSenders
780 .toStub()
781 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
782 proposedContentModification.setTo(id.with);
783 xmppConnectionService.sendIqPacket(
784 id.account,
785 proposedContentModification,
786 (response) -> {
787 if (response.getType() == Iq.Type.RESULT) {
788 Log.d(
789 Config.LOGTAG,
790 id.account.getJid().asBareJid()
791 + ": remote has accepted our upgrade to senders=both");
792 acceptContentAdd(
793 ContentAddition.summary(modifiedSenders), modifiedSenders);
794 } else {
795 Log.d(
796 Config.LOGTAG,
797 id.account.getJid().asBareJid()
798 + ": remote has rejected our upgrade to senders=both");
799 acceptContentAdd(contentAddition, incomingContentAdd);
800 }
801 });
802 } else {
803 acceptContentAdd(contentAddition, incomingContentAdd);
804 }
805 } else {
806 throw new IllegalStateException(
807 "Accepted content add does not match pending content-add");
808 }
809 }
810
811 private void acceptContentAdd(
812 @NonNull final Set<ContentAddition.Summary> contentAddition,
813 final RtpContentMap incomingContentAdd) {
814 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
815 final RtpContentMap modifiedContentMap =
816 getRemoteContentMap().addContent(incomingContentAdd, setup);
817 this.setRemoteContentMap(modifiedContentMap);
818
819 final SessionDescription offer;
820 try {
821 offer = SessionDescription.of(modifiedContentMap, isResponder());
822 } catch (final IllegalArgumentException | NullPointerException e) {
823 Log.d(
824 Config.LOGTAG,
825 id.getAccount().getJid().asBareJid()
826 + ": unable convert offer from content-add to SDP",
827 e);
828 webRTCWrapper.close();
829 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
830 return;
831 }
832 this.incomingContentAdd = null;
833 acceptContentAdd(contentAddition, offer);
834 }
835
836 private void acceptContentAdd(
837 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
838 final org.webrtc.SessionDescription sdp =
839 new org.webrtc.SessionDescription(
840 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
841 try {
842 this.webRTCWrapper.setRemoteDescription(sdp).get();
843
844 // TODO add tracks for 'media' where contentAddition.senders matches
845
846 // TODO if senders.sending(isInitiator())
847
848 this.webRTCWrapper.addTrack(Media.VIDEO);
849
850 // TODO add additional transceivers for recv only cases
851
852 final SessionDescription answer = setLocalSessionDescription();
853 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
854
855 final RtpContentMap contentAcceptMap =
856 rtpContentMap.toContentModification(
857 Collections2.transform(contentAddition, ca -> ca.name));
858
859 Log.d(
860 Config.LOGTAG,
861 id.getAccount().getJid().asBareJid()
862 + ": sending content-accept "
863 + ContentAddition.summary(contentAcceptMap));
864
865 addIceCandidatesFromBlackLog();
866
867 modifyLocalContentMap(rtpContentMap);
868 final ListenableFuture<RtpContentMap> future =
869 prepareOutgoingContentMap(contentAcceptMap);
870 Futures.addCallback(
871 future,
872 new FutureCallback<>() {
873 @Override
874 public void onSuccess(final RtpContentMap rtpContentMap) {
875 sendContentAccept(rtpContentMap);
876 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
877 }
878
879 @Override
880 public void onFailure(@NonNull final Throwable throwable) {
881 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
882 }
883 },
884 MoreExecutors.directExecutor());
885 } catch (final Exception e) {
886 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
887 webRTCWrapper.close();
888 sendSessionTerminate(Reason.FAILED_APPLICATION);
889 }
890 }
891
892 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
893 final Iq iq =
894 contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
895 send(iq);
896 }
897
898 public synchronized void rejectContentAdd() {
899 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
900 if (incomingContentAdd == null) {
901 throw new IllegalStateException("No incoming content add");
902 }
903 this.incomingContentAdd = null;
904 updateEndUserState();
905 rejectContentAdd(incomingContentAdd);
906 }
907
908 private void rejectContentAdd(final RtpContentMap contentMap) {
909 final Iq iq =
910 contentMap
911 .toStub()
912 .toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
913 Log.d(
914 Config.LOGTAG,
915 id.getAccount().getJid().asBareJid()
916 + ": rejecting content "
917 + ContentAddition.summary(contentMap));
918 send(iq);
919 }
920
921 private boolean checkForIceRestart(
922 final Iq jinglePacket, final RtpContentMap rtpContentMap) {
923 final RtpContentMap existing = getRemoteContentMap();
924 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
925 final IceUdpTransportInfo.Credentials newCredentials;
926 try {
927 existingCredentials = existing.getCredentials();
928 newCredentials = rtpContentMap.getDistinctCredentials();
929 } catch (final IllegalStateException e) {
930 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
931 return false;
932 }
933 if (existingCredentials.contains(newCredentials)) {
934 return false;
935 }
936 // TODO an alternative approach is to check if we already got an iq result to our
937 // ICE-restart
938 // and if that's the case we are seeing an answer.
939 // This might be more spec compliant but also more error prone potentially
940 final boolean isSignalStateStable =
941 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
942 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
943 final boolean isOffer = rtpContentMap.emptyCandidates();
944 final RtpContentMap restartContentMap;
945 try {
946 if (isOffer) {
947 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
948 restartContentMap =
949 existing.modifiedCredentials(
950 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
951 } else {
952 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
953 Log.d(
954 Config.LOGTAG,
955 "received confirmation of ICE restart"
956 + newCredentials
957 + " peer_setup="
958 + setup);
959 // DTLS setup attribute needs to be rewritten to reflect current peer state
960 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
961 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
962 }
963 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
964 return isOffer;
965 } else {
966 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
967 respondWithTieBreak(jinglePacket);
968 return true;
969 }
970 } catch (final Exception exception) {
971 respondOk(jinglePacket);
972 final Throwable rootCause = Throwables.getRootCause(exception);
973 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
974 // If this happens a termination is already in progress
975 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
976 return true;
977 }
978 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
979 webRTCWrapper.close();
980 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
981 return true;
982 }
983 }
984
985 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
986 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
987 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
988 throw new IllegalStateException("Invalid peer setup");
989 }
990 return peerSetup;
991 }
992
993 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
994 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
995 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
996 }
997 this.peerDtlsSetup = setup;
998 }
999
1000 private boolean applyIceRestart(
1001 final Iq jinglePacket,
1002 final RtpContentMap restartContentMap,
1003 final boolean isOffer)
1004 throws ExecutionException, InterruptedException {
1005 final SessionDescription sessionDescription =
1006 SessionDescription.of(restartContentMap, isResponder());
1007 final org.webrtc.SessionDescription.Type type =
1008 isOffer
1009 ? org.webrtc.SessionDescription.Type.OFFER
1010 : org.webrtc.SessionDescription.Type.ANSWER;
1011 org.webrtc.SessionDescription sdp =
1012 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1013 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1014 if (isInitiator()) {
1015 // We ignore the offer and respond with tie-break. This will clause the responder
1016 // not to apply the content map
1017 return false;
1018 }
1019 }
1020 webRTCWrapper.setRemoteDescription(sdp).get();
1021 setRemoteContentMap(restartContentMap);
1022 if (isOffer) {
1023 final SessionDescription localSessionDescription = setLocalSessionDescription();
1024 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1025 // We need to respond OK before sending any candidates
1026 respondOk(jinglePacket);
1027 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1028 } else {
1029 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1030 }
1031 return true;
1032 }
1033
1034 private void processCandidates(
1035 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1036 contents) {
1037 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1038 content : contents) {
1039 processCandidate(content);
1040 }
1041 }
1042
1043 private void processCandidate(
1044 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1045 content) {
1046 final RtpContentMap rtpContentMap = getRemoteContentMap();
1047 final List<String> indices = toIdentificationTags(rtpContentMap);
1048 final String sdpMid = content.getKey(); // aka content name
1049 final IceUdpTransportInfo transport = content.getValue().transport;
1050 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1051
1052 // TODO check that credentials remained the same
1053
1054 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1055 final String sdp;
1056 try {
1057 sdp = candidate.toSdpAttribute(credentials.ufrag);
1058 } catch (final IllegalArgumentException e) {
1059 Log.d(
1060 Config.LOGTAG,
1061 id.account.getJid().asBareJid()
1062 + ": ignoring invalid ICE candidate "
1063 + e.getMessage());
1064 continue;
1065 }
1066 final int mLineIndex = indices.indexOf(sdpMid);
1067 if (mLineIndex < 0) {
1068 Log.w(
1069 Config.LOGTAG,
1070 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1071 }
1072 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1073 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1074 this.webRTCWrapper.addIceCandidate(iceCandidate);
1075 }
1076 }
1077
1078 private RtpContentMap getRemoteContentMap() {
1079 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1080 }
1081
1082 private RtpContentMap getLocalContentMap() {
1083 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1084 }
1085
1086 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1087 final Group originalGroup = rtpContentMap.group;
1088 final List<String> identificationTags =
1089 originalGroup == null
1090 ? rtpContentMap.getNames()
1091 : originalGroup.getIdentificationTags();
1092 if (identificationTags.size() == 0) {
1093 Log.w(
1094 Config.LOGTAG,
1095 id.account.getJid().asBareJid()
1096 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1097 }
1098 return identificationTags;
1099 }
1100
1101 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1102 final Jingle jinglePacket, final boolean expectVerification) {
1103 try {
1104 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1105 } catch (final Exception e) {
1106 return Futures.immediateFailedFuture(e);
1107 }
1108 }
1109
1110 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1111 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1112 Log.d(
1113 Config.LOGTAG,
1114 "receiveRtpContentMap("
1115 + receivedContentMap.getClass().getSimpleName()
1116 + ",expectVerification="
1117 + expectVerification
1118 + ")");
1119 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1120 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1121 id.account
1122 .getAxolotlService()
1123 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1124 return Futures.transform(
1125 future,
1126 omemoVerifiedPayload -> {
1127 // TODO test if an exception here triggers a correct abort
1128 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1129 Log.d(
1130 Config.LOGTAG,
1131 id.account.getJid().asBareJid()
1132 + ": received verifiable DTLS fingerprint via "
1133 + omemoVerification);
1134 return omemoVerifiedPayload.getPayload();
1135 },
1136 MoreExecutors.directExecutor());
1137 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1138 return Futures.immediateFailedFuture(
1139 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1140 } else {
1141 return Futures.immediateFuture(receivedContentMap);
1142 }
1143 }
1144
1145 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1146 if (isInitiator()) {
1147 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1148 return;
1149 }
1150 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1151 Futures.addCallback(
1152 future,
1153 new FutureCallback<>() {
1154 @Override
1155 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1156 receiveSessionInitiate(jinglePacket, rtpContentMap);
1157 }
1158
1159 @Override
1160 public void onFailure(@NonNull final Throwable throwable) {
1161 respondOk(jinglePacket);
1162 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1163 }
1164 },
1165 MoreExecutors.directExecutor());
1166 }
1167
1168 private void receiveSessionInitiate(
1169 final Iq jinglePacket, final RtpContentMap contentMap) {
1170 try {
1171 contentMap.requireContentDescriptions();
1172 contentMap.requireDTLSFingerprint(true);
1173 } catch (final RuntimeException e) {
1174 Log.d(
1175 Config.LOGTAG,
1176 id.account.getJid().asBareJid() + ": improperly formatted contents",
1177 Throwables.getRootCause(e));
1178 respondOk(jinglePacket);
1179 sendSessionTerminate(Reason.of(e), e.getMessage());
1180 return;
1181 }
1182 Log.d(
1183 Config.LOGTAG,
1184 "processing session-init with " + contentMap.contents.size() + " contents");
1185 final State target;
1186 if (this.state == State.PROCEED) {
1187 Preconditions.checkState(
1188 proposedMedia != null && proposedMedia.size() > 0,
1189 "proposed media must be set when processing pre-approved session-initiate");
1190 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1191 sendSessionTerminate(
1192 Reason.SECURITY_ERROR,
1193 String.format(
1194 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1195 this.proposedMedia, contentMap.getMedia()));
1196 return;
1197 }
1198 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1199 } else {
1200 target = State.SESSION_INITIALIZED;
1201 setProposedMedia(contentMap.getMedia());
1202 }
1203 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1204 respondOk(jinglePacket);
1205 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1206 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1207 Log.d(
1208 Config.LOGTAG,
1209 id.account.getJid().asBareJid()
1210 + ": automatically accepting session-initiate");
1211 sendSessionAccept();
1212 } else {
1213 Log.d(
1214 Config.LOGTAG,
1215 id.account.getJid().asBareJid()
1216 + ": received not pre-approved session-initiate. start ringing");
1217 startRinging();
1218 }
1219 } else {
1220 Log.d(
1221 Config.LOGTAG,
1222 String.format(
1223 "%s: received session-initiate while in state %s",
1224 id.account.getJid().asBareJid(), state));
1225 terminateWithOutOfOrder(jinglePacket);
1226 }
1227 }
1228
1229 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1230 if (isResponder()) {
1231 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1232 return;
1233 }
1234 final ListenableFuture<RtpContentMap> future =
1235 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1236 Futures.addCallback(
1237 future,
1238 new FutureCallback<>() {
1239 @Override
1240 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1241 receiveSessionAccept(jinglePacket, rtpContentMap);
1242 }
1243
1244 @Override
1245 public void onFailure(@NonNull final Throwable throwable) {
1246 respondOk(jinglePacket);
1247 Log.d(
1248 Config.LOGTAG,
1249 id.account.getJid().asBareJid()
1250 + ": improperly formatted contents in session-accept",
1251 throwable);
1252 webRTCWrapper.close();
1253 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1254 }
1255 },
1256 MoreExecutors.directExecutor());
1257 }
1258
1259 private void receiveSessionAccept(
1260 final Iq jinglePacket, final RtpContentMap contentMap) {
1261 try {
1262 contentMap.requireContentDescriptions();
1263 contentMap.requireDTLSFingerprint();
1264 } catch (final RuntimeException e) {
1265 respondOk(jinglePacket);
1266 Log.d(
1267 Config.LOGTAG,
1268 id.account.getJid().asBareJid()
1269 + ": improperly formatted contents in session-accept",
1270 e);
1271 webRTCWrapper.close();
1272 sendSessionTerminate(Reason.of(e), e.getMessage());
1273 return;
1274 }
1275 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1276 if (!initiatorMedia.equals(contentMap.getMedia())) {
1277 sendSessionTerminate(
1278 Reason.SECURITY_ERROR,
1279 String.format(
1280 "Your session-included included media %s but our session-initiate was %s",
1281 this.proposedMedia, contentMap.getMedia()));
1282 return;
1283 }
1284 Log.d(
1285 Config.LOGTAG,
1286 "processing session-accept with " + contentMap.contents.size() + " contents");
1287 if (transition(State.SESSION_ACCEPTED)) {
1288 respondOk(jinglePacket);
1289 receiveSessionAccept(contentMap);
1290 } else {
1291 Log.d(
1292 Config.LOGTAG,
1293 String.format(
1294 "%s: received session-accept while in state %s",
1295 id.account.getJid().asBareJid(), state));
1296 respondOk(jinglePacket);
1297 }
1298 }
1299
1300 private void receiveSessionAccept(final RtpContentMap contentMap) {
1301 this.responderRtpContentMap = contentMap;
1302 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1303 final SessionDescription sessionDescription;
1304 try {
1305 sessionDescription = SessionDescription.of(contentMap, false);
1306 } catch (final IllegalArgumentException | NullPointerException e) {
1307 Log.d(
1308 Config.LOGTAG,
1309 id.account.getJid().asBareJid()
1310 + ": unable convert offer from session-accept to SDP",
1311 e);
1312 webRTCWrapper.close();
1313 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1314 return;
1315 }
1316 final org.webrtc.SessionDescription answer =
1317 new org.webrtc.SessionDescription(
1318 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1319 try {
1320 this.webRTCWrapper.setRemoteDescription(answer).get();
1321 } catch (final Exception e) {
1322 Log.d(
1323 Config.LOGTAG,
1324 id.account.getJid().asBareJid()
1325 + ": unable to set remote description after receiving session-accept",
1326 Throwables.getRootCause(e));
1327 webRTCWrapper.close();
1328 sendSessionTerminate(
1329 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1330 return;
1331 }
1332 processCandidates(contentMap.contents.entrySet());
1333 }
1334
1335 private void sendSessionAccept() {
1336 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1337 if (rtpContentMap == null) {
1338 throw new IllegalStateException("initiator RTP Content Map has not been set");
1339 }
1340 final SessionDescription offer;
1341 try {
1342 offer = SessionDescription.of(rtpContentMap, true);
1343 } catch (final IllegalArgumentException | NullPointerException e) {
1344 Log.d(
1345 Config.LOGTAG,
1346 id.account.getJid().asBareJid()
1347 + ": unable convert offer from session-initiate to SDP",
1348 e);
1349 webRTCWrapper.close();
1350 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1351 return;
1352 }
1353 sendSessionAccept(rtpContentMap.getMedia(), offer);
1354 }
1355
1356 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1357 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1358 }
1359
1360 private synchronized void sendSessionAccept(
1361 final Set<Media> media,
1362 final SessionDescription offer,
1363 final List<PeerConnection.IceServer> iceServers) {
1364 if (isTerminated()) {
1365 Log.w(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid()
1368 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1369 return;
1370 }
1371 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1372 try {
1373 setupWebRTC(media, iceServers, !includeCandidates);
1374 } catch (final WebRTCWrapper.InitializationException e) {
1375 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1376 webRTCWrapper.close();
1377 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1378 return;
1379 }
1380 final org.webrtc.SessionDescription sdp =
1381 new org.webrtc.SessionDescription(
1382 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1383 try {
1384 this.webRTCWrapper.setRemoteDescription(sdp).get();
1385 addIceCandidatesFromBlackLog();
1386 org.webrtc.SessionDescription webRTCSessionDescription =
1387 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1388 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1389 } catch (final Exception e) {
1390 failureToAcceptSession(e);
1391 }
1392 }
1393
1394 private void failureToAcceptSession(final Throwable throwable) {
1395 if (isTerminated()) {
1396 return;
1397 }
1398 final Throwable rootCause = Throwables.getRootCause(throwable);
1399 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1400 webRTCWrapper.close();
1401 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1402 }
1403
1404 private void failureToPerformAction(
1405 final Jingle.Action action, final Throwable throwable) {
1406 if (isTerminated()) {
1407 return;
1408 }
1409 final Throwable rootCause = Throwables.getRootCause(throwable);
1410 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1411 webRTCWrapper.close();
1412 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1413 }
1414
1415 private void addIceCandidatesFromBlackLog() {
1416 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1417 while ((foo = this.pendingIceCandidates.poll()) != null) {
1418 processCandidate(foo);
1419 Log.d(
1420 Config.LOGTAG,
1421 id.account.getJid().asBareJid() + ": added candidate from back log");
1422 }
1423 }
1424
1425 private void prepareSessionAccept(
1426 final org.webrtc.SessionDescription webRTCSessionDescription,
1427 final boolean includeCandidates) {
1428 final SessionDescription sessionDescription =
1429 SessionDescription.parse(webRTCSessionDescription.description);
1430 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1431 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1432 if (includeCandidates) {
1433 candidates = parseCandidates(sessionDescription);
1434 } else {
1435 candidates = ImmutableMultimap.of();
1436 }
1437 this.responderRtpContentMap = respondingRtpContentMap;
1438 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1439 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1440 prepareOutgoingContentMap(respondingRtpContentMap);
1441 Futures.addCallback(
1442 outgoingContentMapFuture,
1443 new FutureCallback<>() {
1444 @Override
1445 public void onSuccess(final RtpContentMap outgoingContentMap) {
1446 if (includeCandidates) {
1447 Log.d(
1448 Config.LOGTAG,
1449 "including "
1450 + candidates.size()
1451 + " candidates in session accept");
1452 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1453 } else {
1454 sendSessionAccept(outgoingContentMap);
1455 }
1456 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1457 }
1458
1459 @Override
1460 public void onFailure(@NonNull Throwable throwable) {
1461 failureToAcceptSession(throwable);
1462 }
1463 },
1464 MoreExecutors.directExecutor());
1465 }
1466
1467 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1468 if (isTerminated()) {
1469 Log.w(
1470 Config.LOGTAG,
1471 id.account.getJid().asBareJid()
1472 + ": preparing session accept was too slow. already terminated. nothing to do.");
1473 return;
1474 }
1475 transitionOrThrow(State.SESSION_ACCEPTED);
1476 final Iq sessionAccept =
1477 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1478 send(sessionAccept);
1479 }
1480
1481 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1482 final RtpContentMap rtpContentMap) {
1483 if (this.omemoVerification.hasDeviceId()) {
1484 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1485 verifiedPayloadFuture =
1486 id.account
1487 .getAxolotlService()
1488 .encrypt(
1489 rtpContentMap,
1490 id.with,
1491 omemoVerification.getDeviceId());
1492 return Futures.transform(
1493 verifiedPayloadFuture,
1494 verifiedPayload -> {
1495 omemoVerification.setOrEnsureEqual(verifiedPayload);
1496 return verifiedPayload.getPayload();
1497 },
1498 MoreExecutors.directExecutor());
1499 } else {
1500 return Futures.immediateFuture(rtpContentMap);
1501 }
1502 }
1503
1504 synchronized void deliveryMessage(
1505 final Jid from,
1506 final Element message,
1507 final String serverMessageId,
1508 final long timestamp) {
1509 Log.d(
1510 Config.LOGTAG,
1511 id.account.getJid().asBareJid()
1512 + ": delivered message to JingleRtpConnection "
1513 + message);
1514 switch (message.getName()) {
1515 case "propose" -> receivePropose(
1516 from, Propose.upgrade(message), serverMessageId, timestamp);
1517 case "proceed" -> receiveProceed(
1518 from, Proceed.upgrade(message), serverMessageId, timestamp);
1519 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1520 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1521 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1522 }
1523 }
1524
1525 void deliverFailedProceed(final String message) {
1526 Log.d(
1527 Config.LOGTAG,
1528 id.account.getJid().asBareJid()
1529 + ": receive message error for proceed message ("
1530 + Strings.nullToEmpty(message)
1531 + ")");
1532 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1533 webRTCWrapper.close();
1534 Log.d(
1535 Config.LOGTAG,
1536 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1537 this.finish();
1538 }
1539 }
1540
1541 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1542 final boolean originatedFromMyself =
1543 from.asBareJid().equals(id.account.getJid().asBareJid());
1544 if (originatedFromMyself) {
1545 if (transition(State.ACCEPTED)) {
1546 acceptedOnOtherDevice(serverMsgId, timestamp);
1547 } else {
1548 Log.d(
1549 Config.LOGTAG,
1550 id.account.getJid().asBareJid()
1551 + ": unable to transition to accept because already in state="
1552 + this.state);
1553 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1554 }
1555 } else {
1556 Log.d(
1557 Config.LOGTAG,
1558 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1559 }
1560 }
1561
1562 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1563 if (serverMsgId != null) {
1564 this.message.setServerMsgId(serverMsgId);
1565 }
1566 this.message.setTime(timestamp);
1567 this.message.setCarbon(true); // indicate that call was accepted on other device
1568 this.writeLogMessageSuccess(0);
1569 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1570 this.finish();
1571 }
1572
1573 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1574 final boolean originatedFromMyself =
1575 from.asBareJid().equals(id.account.getJid().asBareJid());
1576 // reject from another one of my clients
1577 if (originatedFromMyself) {
1578 receiveRejectFromMyself(serverMsgId, timestamp);
1579 } else if (isInitiator()) {
1580 if (from.equals(id.with)) {
1581 receiveRejectFromResponder();
1582 } else {
1583 Log.d(
1584 Config.LOGTAG,
1585 id.account.getJid()
1586 + ": ignoring reject from "
1587 + from
1588 + " for session with "
1589 + id.with);
1590 }
1591 } else {
1592 Log.d(
1593 Config.LOGTAG,
1594 id.account.getJid()
1595 + ": ignoring reject from "
1596 + from
1597 + " for session with "
1598 + id.with);
1599 }
1600 }
1601
1602 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1603 if (transition(State.REJECTED)) {
1604 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1605 this.finish();
1606 if (serverMsgId != null) {
1607 this.message.setServerMsgId(serverMsgId);
1608 }
1609 this.message.setTime(timestamp);
1610 this.message.setCarbon(true); // indicate that call was rejected on other device
1611 writeLogMessageMissed();
1612 } else {
1613 Log.d(
1614 Config.LOGTAG,
1615 "not able to transition into REJECTED because already in " + this.state);
1616 }
1617 }
1618
1619 private void receiveRejectFromResponder() {
1620 if (isInState(State.PROCEED)) {
1621 Log.d(
1622 Config.LOGTAG,
1623 id.account.getJid()
1624 + ": received reject while still in proceed. callee reconsidered");
1625 closeTransitionLogFinish(State.REJECTED_RACED);
1626 return;
1627 }
1628 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1629 Log.d(
1630 Config.LOGTAG,
1631 id.account.getJid()
1632 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1633 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1634 return;
1635 }
1636 Log.d(
1637 Config.LOGTAG,
1638 id.account.getJid()
1639 + ": ignoring reject from responder because already in state "
1640 + this.state);
1641 }
1642
1643 private void receivePropose(
1644 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1645 final boolean originatedFromMyself =
1646 from.asBareJid().equals(id.account.getJid().asBareJid());
1647 if (originatedFromMyself) {
1648 Log.d(
1649 Config.LOGTAG,
1650 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1651 } else if (transition(
1652 State.PROPOSED,
1653 () -> {
1654 final Collection<RtpDescription> descriptions =
1655 Collections2.transform(
1656 Collections2.filter(
1657 propose.getDescriptions(),
1658 d -> d instanceof RtpDescription),
1659 input -> (RtpDescription) input);
1660 final Collection<Media> media =
1661 Collections2.transform(descriptions, RtpDescription::getMedia);
1662 Preconditions.checkState(
1663 !media.contains(Media.UNKNOWN),
1664 "RTP descriptions contain unknown media");
1665 Log.d(
1666 Config.LOGTAG,
1667 id.account.getJid().asBareJid()
1668 + ": received session proposal from "
1669 + from
1670 + " for "
1671 + media);
1672 this.setProposedMedia(Sets.newHashSet(media));
1673 })) {
1674 if (serverMsgId != null) {
1675 this.message.setServerMsgId(serverMsgId);
1676 }
1677 this.message.setTime(timestamp);
1678 startRinging();
1679 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1680 sendJingleMessage("ringing");
1681 }
1682 } else {
1683 Log.d(
1684 Config.LOGTAG,
1685 id.account.getJid()
1686 + ": ignoring session proposal because already in "
1687 + state);
1688 }
1689 }
1690
1691 private void startRinging() {
1692 this.callIntegration.setRinging();
1693 Log.d(
1694 Config.LOGTAG,
1695 id.account.getJid().asBareJid()
1696 + ": received call from "
1697 + id.with
1698 + ". start ringing");
1699 ringingTimeoutFuture =
1700 jingleConnectionManager.schedule(
1701 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1702 if (CallIntegration.selfManaged(xmppConnectionService)) {
1703 return;
1704 }
1705 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1706 }
1707
1708 private synchronized void ringingTimeout() {
1709 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1710 switch (this.state) {
1711 case PROPOSED -> {
1712 message.markUnread();
1713 rejectCallFromProposed();
1714 }
1715 case SESSION_INITIALIZED -> {
1716 message.markUnread();
1717 rejectCallFromSessionInitiate();
1718 }
1719 }
1720 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1721 }
1722
1723 private void cancelRingingTimeout() {
1724 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1725 if (future != null && !future.isCancelled()) {
1726 future.cancel(false);
1727 }
1728 }
1729
1730 private void receiveProceed(
1731 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1732 final Set<Media> media =
1733 Preconditions.checkNotNull(
1734 this.proposedMedia, "Proposed media has to be set before handling proceed");
1735 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1736 if (from.equals(id.with)) {
1737 if (isInitiator()) {
1738 if (transition(State.PROCEED)) {
1739 if (serverMsgId != null) {
1740 this.message.setServerMsgId(serverMsgId);
1741 }
1742 this.message.setTime(timestamp);
1743 final Integer remoteDeviceId = proceed.getDeviceId();
1744 if (isOmemoEnabled()) {
1745 this.omemoVerification.setDeviceId(remoteDeviceId);
1746 } else {
1747 if (remoteDeviceId != null) {
1748 Log.d(
1749 Config.LOGTAG,
1750 id.account.getJid().asBareJid()
1751 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1752 }
1753 this.omemoVerification.setDeviceId(null);
1754 }
1755 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1756 } else {
1757 Log.d(
1758 Config.LOGTAG,
1759 String.format(
1760 "%s: ignoring proceed because already in %s",
1761 id.account.getJid().asBareJid(), this.state));
1762 }
1763 } else {
1764 Log.d(
1765 Config.LOGTAG,
1766 String.format(
1767 "%s: ignoring proceed because we were not initializing",
1768 id.account.getJid().asBareJid()));
1769 }
1770 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1771 if (transition(State.ACCEPTED)) {
1772 Log.d(
1773 Config.LOGTAG,
1774 id.account.getJid().asBareJid()
1775 + ": moved session with "
1776 + id.with
1777 + " into state accepted after received carbon copied proceed");
1778 acceptedOnOtherDevice(serverMsgId, timestamp);
1779 }
1780 } else {
1781 Log.d(
1782 Config.LOGTAG,
1783 String.format(
1784 "%s: ignoring proceed from %s. was expected from %s",
1785 id.account.getJid().asBareJid(), from, id.with));
1786 }
1787 }
1788
1789 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1790 if (from.equals(id.with)) {
1791 final State target =
1792 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1793 if (transition(target)) {
1794 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1795 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1796 Log.d(
1797 Config.LOGTAG,
1798 id.account.getJid().asBareJid()
1799 + ": session with "
1800 + id.with
1801 + " has been retracted (serverMsgId="
1802 + serverMsgId
1803 + ")");
1804 if (serverMsgId != null) {
1805 this.message.setServerMsgId(serverMsgId);
1806 }
1807 this.message.setTime(timestamp);
1808 if (target == State.RETRACTED) {
1809 this.message.markUnread();
1810 }
1811 writeLogMessageMissed();
1812 finish();
1813 } else {
1814 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1815 }
1816 } else {
1817 // TODO parse retract from self
1818 Log.d(
1819 Config.LOGTAG,
1820 id.account.getJid().asBareJid()
1821 + ": received retract from "
1822 + from
1823 + ". expected retract from"
1824 + id.with
1825 + ". ignoring");
1826 }
1827 }
1828
1829 public void sendSessionInitiate() {
1830 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1831 }
1832
1833 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1834 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1835 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1836 }
1837
1838 private synchronized void sendSessionInitiate(
1839 final Set<Media> media,
1840 final State targetState,
1841 final List<PeerConnection.IceServer> iceServers) {
1842 if (isTerminated()) {
1843 Log.w(
1844 Config.LOGTAG,
1845 id.account.getJid().asBareJid()
1846 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1847 return;
1848 }
1849 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1850 try {
1851 setupWebRTC(media, iceServers, !includeCandidates);
1852 } catch (final WebRTCWrapper.InitializationException e) {
1853 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1854 webRTCWrapper.close();
1855 sendRetract(Reason.ofThrowable(e));
1856 return;
1857 }
1858 try {
1859 org.webrtc.SessionDescription webRTCSessionDescription =
1860 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1861 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1862 } catch (final Exception e) {
1863 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1864 // exceptions
1865 failureToInitiateSession(e, targetState);
1866 }
1867 }
1868
1869 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1870 if (isTerminated()) {
1871 return;
1872 }
1873 Log.d(
1874 Config.LOGTAG,
1875 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1876 Throwables.getRootCause(throwable));
1877 webRTCWrapper.close();
1878 final Reason reason = Reason.ofThrowable(throwable);
1879 if (isInState(targetState)) {
1880 sendSessionTerminate(reason, throwable.getMessage());
1881 } else {
1882 sendRetract(reason);
1883 }
1884 }
1885
1886 private void sendRetract(final Reason reason) {
1887 // TODO embed reason into retract
1888 sendJingleMessage("retract", id.with.asBareJid());
1889 transitionOrThrow(reasonToState(reason));
1890 this.finish();
1891 }
1892
1893 private void prepareSessionInitiate(
1894 final org.webrtc.SessionDescription webRTCSessionDescription,
1895 final boolean includeCandidates,
1896 final State targetState) {
1897 final SessionDescription sessionDescription =
1898 SessionDescription.parse(webRTCSessionDescription.description);
1899 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1900 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1901 if (includeCandidates) {
1902 candidates = parseCandidates(sessionDescription);
1903 } else {
1904 candidates = ImmutableMultimap.of();
1905 }
1906 this.initiatorRtpContentMap = rtpContentMap;
1907 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1908 encryptSessionInitiate(rtpContentMap);
1909 Futures.addCallback(
1910 outgoingContentMapFuture,
1911 new FutureCallback<>() {
1912 @Override
1913 public void onSuccess(final RtpContentMap outgoingContentMap) {
1914 if (includeCandidates) {
1915 Log.d(
1916 Config.LOGTAG,
1917 "including "
1918 + candidates.size()
1919 + " candidates in session initiate");
1920 sendSessionInitiate(
1921 outgoingContentMap.withCandidates(candidates), targetState);
1922 } else {
1923 sendSessionInitiate(outgoingContentMap, targetState);
1924 }
1925 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1926 }
1927
1928 @Override
1929 public void onFailure(@NonNull final Throwable throwable) {
1930 failureToInitiateSession(throwable, targetState);
1931 }
1932 },
1933 MoreExecutors.directExecutor());
1934 }
1935
1936 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1937 if (isTerminated()) {
1938 Log.w(
1939 Config.LOGTAG,
1940 id.account.getJid().asBareJid()
1941 + ": preparing session was too slow. already terminated. nothing to do.");
1942 return;
1943 }
1944 this.transitionOrThrow(targetState);
1945 final Iq sessionInitiate =
1946 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1947 send(sessionInitiate);
1948 }
1949
1950 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1951 final RtpContentMap rtpContentMap) {
1952 if (this.omemoVerification.hasDeviceId()) {
1953 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1954 verifiedPayloadFuture =
1955 id.account
1956 .getAxolotlService()
1957 .encrypt(
1958 rtpContentMap,
1959 id.with,
1960 omemoVerification.getDeviceId());
1961 final ListenableFuture<RtpContentMap> future =
1962 Futures.transform(
1963 verifiedPayloadFuture,
1964 verifiedPayload -> {
1965 omemoVerification.setSessionFingerprint(
1966 verifiedPayload.getFingerprint());
1967 return verifiedPayload.getPayload();
1968 },
1969 MoreExecutors.directExecutor());
1970 if (Config.REQUIRE_RTP_VERIFICATION) {
1971 return future;
1972 }
1973 return Futures.catching(
1974 future,
1975 CryptoFailedException.class,
1976 e -> {
1977 Log.w(
1978 Config.LOGTAG,
1979 id.account.getJid().asBareJid()
1980 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1981 e);
1982 return rtpContentMap;
1983 },
1984 MoreExecutors.directExecutor());
1985 } else {
1986 return Futures.immediateFuture(rtpContentMap);
1987 }
1988 }
1989
1990 protected void sendSessionTerminate(final Reason reason) {
1991 sendSessionTerminate(reason, null);
1992 }
1993
1994 protected void sendSessionTerminate(final Reason reason, final String text) {
1995 sendSessionTerminate(reason, text, this::writeLogMessage);
1996 sendJingleMessageFinish(reason);
1997 }
1998
1999 private void sendTransportInfo(
2000 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2001 final RtpContentMap transportInfo;
2002 try {
2003 final RtpContentMap rtpContentMap =
2004 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2005 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2006 } catch (final Exception e) {
2007 Log.d(
2008 Config.LOGTAG,
2009 id.account.getJid().asBareJid()
2010 + ": unable to prepare transport-info from candidate for content="
2011 + contentName);
2012 return;
2013 }
2014 final Iq iq =
2015 transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2016 send(iq);
2017 }
2018
2019 public RtpEndUserState getEndUserState() {
2020 switch (this.state) {
2021 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2022 if (isInitiator()) {
2023 return RtpEndUserState.RINGING;
2024 } else {
2025 return RtpEndUserState.INCOMING_CALL;
2026 }
2027 }
2028 case PROCEED -> {
2029 if (isInitiator()) {
2030 return RtpEndUserState.RINGING;
2031 } else {
2032 return RtpEndUserState.ACCEPTING_CALL;
2033 }
2034 }
2035 case SESSION_INITIALIZED_PRE_APPROVED -> {
2036 if (isInitiator()) {
2037 return RtpEndUserState.RINGING;
2038 } else {
2039 return RtpEndUserState.CONNECTING;
2040 }
2041 }
2042 case SESSION_ACCEPTED -> {
2043 final ContentAddition ca = getPendingContentAddition();
2044 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2045 return RtpEndUserState.INCOMING_CONTENT_ADD;
2046 }
2047 return getPeerConnectionStateAsEndUserState();
2048 }
2049 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2050 if (isInitiator()) {
2051 return RtpEndUserState.DECLINED_OR_BUSY;
2052 } else {
2053 return RtpEndUserState.ENDED;
2054 }
2055 }
2056 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2057 return RtpEndUserState.ENDED;
2058 }
2059 case RETRACTED_RACED -> {
2060 if (isInitiator()) {
2061 return RtpEndUserState.ENDED;
2062 } else {
2063 return RtpEndUserState.RETRACTED;
2064 }
2065 }
2066 case TERMINATED_CONNECTIVITY_ERROR -> {
2067 return zeroDuration()
2068 ? RtpEndUserState.CONNECTIVITY_ERROR
2069 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2070 }
2071 case TERMINATED_APPLICATION_FAILURE -> {
2072 return RtpEndUserState.APPLICATION_ERROR;
2073 }
2074 case TERMINATED_SECURITY_ERROR -> {
2075 return RtpEndUserState.SECURITY_ERROR;
2076 }
2077 }
2078 throw new IllegalStateException(
2079 String.format("%s has no equivalent EndUserState", this.state));
2080 }
2081
2082 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2083 final PeerConnection.PeerConnectionState state;
2084 try {
2085 state = webRTCWrapper.getState();
2086 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2087 // We usually close the WebRTCWrapper *before* transitioning so we might still
2088 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2089 return RtpEndUserState.ENDING_CALL;
2090 }
2091 return switch (state) {
2092 case CONNECTED -> RtpEndUserState.CONNECTED;
2093 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2094 case CLOSED -> RtpEndUserState.ENDING_CALL;
2095 default -> zeroDuration()
2096 ? RtpEndUserState.CONNECTIVITY_ERROR
2097 : RtpEndUserState.RECONNECTING;
2098 };
2099 }
2100
2101 private boolean isPeerConnectionConnected() {
2102 try {
2103 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2104 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2105 return false;
2106 }
2107 }
2108
2109 private void updateCallIntegrationState() {
2110 switch (this.state) {
2111 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2112 if (isInitiator()) {
2113 this.callIntegration.setDialing();
2114 } else {
2115 this.callIntegration.setRinging();
2116 }
2117 }
2118 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2119 if (isInitiator()) {
2120 this.callIntegration.setDialing();
2121 } else {
2122 this.callIntegration.setInitialized();
2123 }
2124 }
2125 case SESSION_ACCEPTED -> {
2126 if (isPeerConnectionConnected()) {
2127 this.callIntegration.setActive();
2128 } else {
2129 this.callIntegration.setInitialized();
2130 }
2131 }
2132 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2133 if (isInitiator()) {
2134 this.callIntegration.busy();
2135 } else {
2136 this.callIntegration.rejected();
2137 }
2138 }
2139 case TERMINATED_SUCCESS -> this.callIntegration.success();
2140 case ACCEPTED -> this.callIntegration.accepted();
2141 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2142 .retracted();
2143 case TERMINATED_CONNECTIVITY_ERROR,
2144 TERMINATED_APPLICATION_FAILURE,
2145 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2146 default -> throw new IllegalStateException(
2147 String.format("%s is not handled", this.state));
2148 }
2149 }
2150
2151 public ContentAddition getPendingContentAddition() {
2152 final RtpContentMap in = this.incomingContentAdd;
2153 final RtpContentMap out = this.outgoingContentAdd;
2154 if (out != null) {
2155 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2156 } else if (in != null) {
2157 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2158 } else {
2159 return null;
2160 }
2161 }
2162
2163 public Set<Media> getMedia() {
2164 final State current = getState();
2165 if (current == State.NULL) {
2166 if (isInitiator()) {
2167 return Preconditions.checkNotNull(
2168 this.proposedMedia, "RTP connection has not been initialized properly");
2169 }
2170 throw new IllegalStateException("RTP connection has not been initialized yet");
2171 }
2172 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2173 return Preconditions.checkNotNull(
2174 this.proposedMedia, "RTP connection has not been initialized properly");
2175 }
2176 final RtpContentMap localContentMap = getLocalContentMap();
2177 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2178 if (localContentMap != null) {
2179 return localContentMap.getMedia();
2180 } else if (initiatorContentMap != null) {
2181 return initiatorContentMap.getMedia();
2182 } else if (isTerminated()) {
2183 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2184 } else {
2185 return Preconditions.checkNotNull(
2186 this.proposedMedia, "RTP connection has not been initialized properly");
2187 }
2188 }
2189
2190 public boolean isVerified() {
2191 final String fingerprint = this.omemoVerification.getFingerprint();
2192 if (fingerprint == null) {
2193 return false;
2194 }
2195 final FingerprintStatus status =
2196 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2197 return status != null && status.isVerified();
2198 }
2199
2200 public boolean addMedia(final Media media) {
2201 final Set<Media> currentMedia = getMedia();
2202 if (currentMedia.contains(media)) {
2203 throw new IllegalStateException(String.format("%s has already been proposed", media));
2204 }
2205 // TODO add state protection - can only add while ACCEPTED or so
2206 Log.d(Config.LOGTAG, "adding media: " + media);
2207 return webRTCWrapper.addTrack(media);
2208 }
2209
2210 public synchronized void acceptCall() {
2211 switch (this.state) {
2212 case PROPOSED -> {
2213 cancelRingingTimeout();
2214 acceptCallFromProposed();
2215 }
2216 case SESSION_INITIALIZED -> {
2217 cancelRingingTimeout();
2218 acceptCallFromSessionInitialized();
2219 }
2220 case ACCEPTED -> Log.w(
2221 Config.LOGTAG,
2222 id.account.getJid().asBareJid()
2223 + ": the call has already been accepted with another client. UI was just lagging behind");
2224 case PROCEED, SESSION_ACCEPTED -> Log.w(
2225 Config.LOGTAG,
2226 id.account.getJid().asBareJid()
2227 + ": the call has already been accepted. user probably double tapped the UI");
2228 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2229 }
2230 }
2231
2232 public synchronized void rejectCall() {
2233 if (isTerminated()) {
2234 Log.w(
2235 Config.LOGTAG,
2236 id.account.getJid().asBareJid()
2237 + ": received rejectCall() when session has already been terminated. nothing to do");
2238 return;
2239 }
2240 switch (this.state) {
2241 case PROPOSED -> rejectCallFromProposed();
2242 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2243 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2244 }
2245 }
2246
2247 public synchronized void integrationFailure() {
2248 final var state = getState();
2249 if (state == State.PROPOSED) {
2250 Log.e(
2251 Config.LOGTAG,
2252 id.account.getJid().asBareJid()
2253 + ": failed call integration in state proposed");
2254 rejectCallFromProposed();
2255 } else if (state == State.SESSION_INITIALIZED) {
2256 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2257 this.webRTCWrapper.close();
2258 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2259 } else {
2260 throw new IllegalStateException(
2261 String.format("Can not fail integration in state %s", state));
2262 }
2263 }
2264
2265 public synchronized void endCall() {
2266 if (isTerminated()) {
2267 Log.w(
2268 Config.LOGTAG,
2269 id.account.getJid().asBareJid()
2270 + ": received endCall() when session has already been terminated. nothing to do");
2271 return;
2272 }
2273 if (isInState(State.PROPOSED) && isResponder()) {
2274 rejectCallFromProposed();
2275 return;
2276 }
2277 if (isInState(State.PROCEED)) {
2278 if (isInitiator()) {
2279 retractFromProceed();
2280 } else {
2281 rejectCallFromProceed();
2282 }
2283 return;
2284 }
2285 if (isInitiator()
2286 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2287 this.webRTCWrapper.close();
2288 sendSessionTerminate(Reason.CANCEL);
2289 return;
2290 }
2291 if (isInState(State.SESSION_INITIALIZED)) {
2292 rejectCallFromSessionInitiate();
2293 return;
2294 }
2295 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2296 this.webRTCWrapper.close();
2297 sendSessionTerminate(Reason.SUCCESS);
2298 return;
2299 }
2300 if (isInState(
2301 State.TERMINATED_APPLICATION_FAILURE,
2302 State.TERMINATED_CONNECTIVITY_ERROR,
2303 State.TERMINATED_DECLINED_OR_BUSY)) {
2304 Log.d(
2305 Config.LOGTAG,
2306 "ignoring request to end call because already in state " + this.state);
2307 return;
2308 }
2309 throw new IllegalStateException(
2310 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2311 }
2312
2313 private void retractFromProceed() {
2314 Log.d(Config.LOGTAG, "retract from proceed");
2315 this.sendJingleMessage("retract");
2316 closeTransitionLogFinish(State.RETRACTED_RACED);
2317 }
2318
2319 private void closeTransitionLogFinish(final State state) {
2320 this.webRTCWrapper.close();
2321 transitionOrThrow(state);
2322 writeLogMessage(state);
2323 finish();
2324 }
2325
2326 private void setupWebRTC(
2327 final Set<Media> media,
2328 final List<PeerConnection.IceServer> iceServers,
2329 final boolean trickle)
2330 throws WebRTCWrapper.InitializationException {
2331 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2332 this.webRTCWrapper.setup(this.xmppConnectionService);
2333 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2334 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2335 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2336 }
2337
2338 private void acceptCallFromProposed() {
2339 transitionOrThrow(State.PROCEED);
2340 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2341 this.callIntegration.startAudioRouting();
2342 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2343 this.sendJingleMessage("proceed");
2344 }
2345
2346 private void rejectCallFromProposed() {
2347 transitionOrThrow(State.REJECTED);
2348 writeLogMessageMissed();
2349 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2350 this.sendJingleMessage("reject");
2351 finish();
2352 }
2353
2354 private void rejectCallFromProceed() {
2355 this.sendJingleMessage("reject");
2356 closeTransitionLogFinish(State.REJECTED_RACED);
2357 }
2358
2359 private void rejectCallFromSessionInitiate() {
2360 webRTCWrapper.close();
2361 sendSessionTerminate(Reason.DECLINE);
2362 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2363 }
2364
2365 private void sendJingleMessage(final String action) {
2366 sendJingleMessage(action, id.with);
2367 }
2368
2369 private void sendJingleMessage(final String action, final Jid to) {
2370 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2371 messagePacket.setType(im.conversations.android.xmpp.model.stanza.Message.Type.CHAT); // we want to carbon copy those
2372 messagePacket.setTo(to);
2373 final Element intent =
2374 messagePacket
2375 .addChild(action, Namespace.JINGLE_MESSAGE)
2376 .setAttribute("id", id.sessionId);
2377 if ("proceed".equals(action)) {
2378 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2379 if (isOmemoEnabled()) {
2380 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2381 final Element device =
2382 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2383 device.setAttribute("id", deviceId);
2384 }
2385 }
2386 messagePacket.addChild("store", "urn:xmpp:hints");
2387 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2388 }
2389
2390 private void sendJingleMessageFinish(final Reason reason) {
2391 final var account = id.getAccount();
2392 final var messagePacket =
2393 xmppConnectionService
2394 .getMessageGenerator()
2395 .sessionFinish(id.with, id.sessionId, reason);
2396 xmppConnectionService.sendMessagePacket(account, messagePacket);
2397 }
2398
2399 private boolean isOmemoEnabled() {
2400 final Conversational conversational = message.getConversation();
2401 if (conversational instanceof Conversation) {
2402 return ((Conversation) conversational).getNextEncryption()
2403 == Message.ENCRYPTION_AXOLOTL;
2404 }
2405 return false;
2406 }
2407
2408 private void acceptCallFromSessionInitialized() {
2409 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2410 this.callIntegration.startAudioRouting();
2411 sendSessionAccept();
2412 }
2413
2414 @Override
2415 protected synchronized boolean transition(final State target, final Runnable runnable) {
2416 if (super.transition(target, runnable)) {
2417 updateEndUserState();
2418 updateOngoingCallNotification();
2419 return true;
2420 } else {
2421 return false;
2422 }
2423 }
2424
2425 @Override
2426 public void onIceCandidate(final IceCandidate iceCandidate) {
2427 final RtpContentMap rtpContentMap =
2428 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2429 final IceUdpTransportInfo.Credentials credentials;
2430 try {
2431 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2432 } catch (final IllegalArgumentException e) {
2433 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2434 return;
2435 }
2436 final String uFrag = credentials.ufrag;
2437 final IceUdpTransportInfo.Candidate candidate =
2438 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2439 if (candidate == null) {
2440 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2441 return;
2442 }
2443 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2444 sendTransportInfo(iceCandidate.sdpMid, candidate);
2445 }
2446
2447 @Override
2448 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2449 Log.d(
2450 Config.LOGTAG,
2451 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2452 this.stateHistory.add(newState);
2453 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2454 this.sessionDuration.start();
2455 updateOngoingCallNotification();
2456 } else if (this.sessionDuration.isRunning()) {
2457 this.sessionDuration.stop();
2458 updateOngoingCallNotification();
2459 }
2460
2461 final boolean neverConnected =
2462 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2463
2464 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2465 if (neverConnected) {
2466 if (isTerminated()) {
2467 Log.d(
2468 Config.LOGTAG,
2469 id.account.getJid().asBareJid()
2470 + ": not sending session-terminate after connectivity error because session is already in state "
2471 + this.state);
2472 return;
2473 }
2474 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2475 return;
2476 } else {
2477 this.restartIce();
2478 }
2479 }
2480 updateEndUserState();
2481 }
2482
2483 private void restartIce() {
2484 this.stateHistory.clear();
2485 this.webRTCWrapper.restartIceAsync();
2486 }
2487
2488 @Override
2489 public void onRenegotiationNeeded() {
2490 this.webRTCWrapper.execute(this::renegotiate);
2491 }
2492
2493 private synchronized void renegotiate() {
2494 final SessionDescription sessionDescription;
2495 try {
2496 sessionDescription = setLocalSessionDescription();
2497 } catch (final Exception e) {
2498 final Throwable cause = Throwables.getRootCause(e);
2499 webRTCWrapper.close();
2500 if (isTerminated()) {
2501 Log.d(
2502 Config.LOGTAG,
2503 "failed to renegotiate. session was already terminated",
2504 cause);
2505 return;
2506 }
2507 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2508 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2509 return;
2510 }
2511 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2512 final RtpContentMap currentContentMap = getLocalContentMap();
2513 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2514 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2515
2516 Log.d(
2517 Config.LOGTAG,
2518 id.getAccount().getJid().asBareJid()
2519 + ": renegotiate. iceRestart="
2520 + iceRestart
2521 + " content id diff="
2522 + diff);
2523
2524 if (diff.hasModifications() && iceRestart) {
2525 webRTCWrapper.close();
2526 sendSessionTerminate(
2527 Reason.FAILED_APPLICATION,
2528 "WebRTC unexpectedly tried to modify content and transport at once");
2529 return;
2530 }
2531
2532 if (iceRestart) {
2533 initiateIceRestart(rtpContentMap);
2534 return;
2535 } else if (diff.isEmpty()) {
2536 Log.d(
2537 Config.LOGTAG,
2538 "renegotiation. nothing to do. SignalingState="
2539 + this.webRTCWrapper.getSignalingState());
2540 }
2541
2542 if (diff.added.isEmpty()) {
2543 return;
2544 }
2545 modifyLocalContentMap(rtpContentMap);
2546 sendContentAdd(rtpContentMap, diff.added);
2547 }
2548
2549 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2550 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2551 final Iq iq =
2552 transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2553 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2554 iq.setTo(id.with);
2555 xmppConnectionService.sendIqPacket(
2556 id.account,
2557 iq,
2558 (response) -> {
2559 if (response.getType() == Iq.Type.RESULT) {
2560 Log.d(Config.LOGTAG, "received success to our ice restart");
2561 setLocalContentMap(rtpContentMap);
2562 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2563 return;
2564 }
2565 if (response.getType() == Iq.Type.ERROR) {
2566 if (isTieBreak(response)) {
2567 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2568 return;
2569 }
2570 handleIqErrorResponse(response);
2571 }
2572 if (response.getType() == Iq.Type.TIMEOUT) {
2573 handleIqTimeoutResponse(response);
2574 }
2575 });
2576 }
2577
2578 private boolean isTieBreak(final Iq response) {
2579 final Element error = response.findChild("error");
2580 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2581 }
2582
2583 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2584 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2585 this.outgoingContentAdd = contentAdd;
2586 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2587 prepareOutgoingContentMap(contentAdd);
2588 Futures.addCallback(
2589 outgoingContentMapFuture,
2590 new FutureCallback<>() {
2591 @Override
2592 public void onSuccess(final RtpContentMap outgoingContentMap) {
2593 sendContentAdd(outgoingContentMap);
2594 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2595 }
2596
2597 @Override
2598 public void onFailure(@NonNull Throwable throwable) {
2599 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2600 }
2601 },
2602 MoreExecutors.directExecutor());
2603 }
2604
2605 private void sendContentAdd(final RtpContentMap contentAdd) {
2606
2607 final Iq iq =
2608 contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2609 iq.setTo(id.with);
2610 xmppConnectionService.sendIqPacket(
2611 id.account,
2612 iq,
2613 (response) -> {
2614 if (response.getType() == Iq.Type.RESULT) {
2615 Log.d(
2616 Config.LOGTAG,
2617 id.getAccount().getJid().asBareJid()
2618 + ": received ACK to our content-add");
2619 return;
2620 }
2621 if (response.getType() == Iq.Type.ERROR) {
2622 if (isTieBreak(response)) {
2623 this.outgoingContentAdd = null;
2624 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2625 return;
2626 }
2627 handleIqErrorResponse(response);
2628 }
2629 if (response.getType() == Iq.Type.TIMEOUT) {
2630 handleIqTimeoutResponse(response);
2631 }
2632 });
2633 }
2634
2635 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2636 if (isInitiator()) {
2637 this.initiatorRtpContentMap = rtpContentMap;
2638 } else {
2639 this.responderRtpContentMap = rtpContentMap;
2640 }
2641 }
2642
2643 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2644 if (isInitiator()) {
2645 this.responderRtpContentMap = rtpContentMap;
2646 } else {
2647 this.initiatorRtpContentMap = rtpContentMap;
2648 }
2649 }
2650
2651 // this method is to be used for content map modifications that modify media
2652 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2653 final RtpContentMap activeContents = rtpContentMap.activeContents();
2654 setLocalContentMap(activeContents);
2655 this.callIntegration.setAudioDeviceWhenAvailable(
2656 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2657 updateEndUserState();
2658 }
2659
2660 private SessionDescription setLocalSessionDescription()
2661 throws ExecutionException, InterruptedException {
2662 final org.webrtc.SessionDescription sessionDescription =
2663 this.webRTCWrapper.setLocalDescription(false).get();
2664 return SessionDescription.parse(sessionDescription.description);
2665 }
2666
2667 private void closeWebRTCSessionAfterFailedConnection() {
2668 this.webRTCWrapper.close();
2669 synchronized (this) {
2670 if (isTerminated()) {
2671 Log.d(
2672 Config.LOGTAG,
2673 id.account.getJid().asBareJid()
2674 + ": no need to send session-terminate after failed connection. Other party already did");
2675 return;
2676 }
2677 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2678 }
2679 }
2680
2681 public boolean zeroDuration() {
2682 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2683 }
2684
2685 public long getCallDuration() {
2686 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2687 }
2688
2689 @Override
2690 public CallIntegration getCallIntegration() {
2691 return this.callIntegration;
2692 }
2693
2694 public boolean isMicrophoneEnabled() {
2695 return webRTCWrapper.isMicrophoneEnabled();
2696 }
2697
2698 public boolean setMicrophoneEnabled(final boolean enabled) {
2699 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2700 }
2701
2702 public boolean isVideoEnabled() {
2703 return webRTCWrapper.isVideoEnabled();
2704 }
2705
2706 public void setVideoEnabled(final boolean enabled) {
2707 webRTCWrapper.setVideoEnabled(enabled);
2708 }
2709
2710 public boolean isCameraSwitchable() {
2711 return webRTCWrapper.isCameraSwitchable();
2712 }
2713
2714 public boolean isFrontCamera() {
2715 return webRTCWrapper.isFrontCamera();
2716 }
2717
2718 public ListenableFuture<Boolean> switchCamera() {
2719 return webRTCWrapper.switchCamera();
2720 }
2721
2722 @Override
2723 public synchronized void onCallIntegrationShowIncomingCallUi() {
2724 if (isTerminated()) {
2725 // there might be race conditions with the call integration service invoking this
2726 // callback when the rtp session has already ended.
2727 Log.w(
2728 Config.LOGTAG,
2729 "CallIntegration requested incoming call UI but session was already terminated");
2730 return;
2731 }
2732 // TODO apparently this can be called too early as well?
2733 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2734 }
2735
2736 @Override
2737 public void onCallIntegrationDisconnect() {
2738 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2739 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2740 rejectCall();
2741 } else {
2742 endCall();
2743 }
2744 }
2745
2746 @Override
2747 public void onCallIntegrationReject() {
2748 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2749 try {
2750 rejectCall();
2751 } catch (final IllegalStateException e) {
2752 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2753 }
2754 }
2755
2756 @Override
2757 public void onCallIntegrationAnswer() {
2758 // we need to start the UI to a) show it and b) be able to ask for permissions
2759 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2760 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2761 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2762 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2763 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2764 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2765 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2766 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2767 xmppConnectionService.startActivity(intent);
2768 }
2769
2770 @Override
2771 public void onCallIntegrationSilence() {
2772 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2773 }
2774
2775 @Override
2776 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2777 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2778 // that was previous disabled by the user. A proper implementation would probably be to
2779 // track user choice and enable the microphone with a userEnabled() &&
2780 // callIntegration.isMicrophoneEnabled() condition
2781 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2782 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2783 }
2784
2785 @Override
2786 public void onAudioDeviceChanged(
2787 final CallIntegration.AudioDevice selectedAudioDevice,
2788 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2789 Log.d(
2790 Config.LOGTAG,
2791 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2792 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2793 selectedAudioDevice, availableAudioDevices);
2794 }
2795
2796 private void updateEndUserState() {
2797 final RtpEndUserState endUserState = getEndUserState();
2798 this.updateCallIntegrationState();
2799 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2800 id.account, id.with, id.sessionId, endUserState);
2801 }
2802
2803 private void updateOngoingCallNotification() {
2804 final State state = this.state;
2805 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2806 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2807 && isResponder()) {
2808 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2809 xmppConnectionService.removeOngoingCall();
2810 return;
2811 }
2812 final boolean reconnecting;
2813 if (state == State.SESSION_ACCEPTED) {
2814 reconnecting =
2815 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2816 } else {
2817 reconnecting = false;
2818 }
2819 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2820 } else {
2821 xmppConnectionService.removeOngoingCall();
2822 }
2823 }
2824
2825 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2826 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2827 final Iq request = new Iq(Iq.Type.GET);
2828 request.setTo(id.account.getDomain());
2829 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2830 xmppConnectionService.sendIqPacket(
2831 id.account,
2832 request,
2833 (response) -> {
2834 final var iceServers = IceServers.parse(response);
2835 if (iceServers.isEmpty()) {
2836 Log.w(
2837 Config.LOGTAG,
2838 id.account.getJid().asBareJid()
2839 + ": no ICE server found "
2840 + response);
2841 }
2842 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2843 });
2844 } else {
2845 Log.w(
2846 Config.LOGTAG,
2847 id.account.getJid().asBareJid() + ": has no external service discovery");
2848 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2849 }
2850 }
2851
2852 @Override
2853 protected void terminateTransport() {
2854 this.webRTCWrapper.close();
2855 }
2856
2857 @Override
2858 protected void finish() {
2859 if (isTerminated()) {
2860 this.cancelRingingTimeout();
2861 this.callIntegration.verifyDisconnected();
2862 this.webRTCWrapper.verifyClosed();
2863 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2864 super.finish();
2865 } else {
2866 throw new IllegalStateException(
2867 String.format("Unable to call finish from %s", this.state));
2868 }
2869 }
2870
2871 private void writeLogMessage(final State state) {
2872 final long duration = getCallDuration();
2873 if (state == State.TERMINATED_SUCCESS
2874 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2875 writeLogMessageSuccess(duration);
2876 } else {
2877 writeLogMessageMissed();
2878 }
2879 }
2880
2881 private void writeLogMessageSuccess(final long duration) {
2882 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2883 this.writeMessage();
2884 }
2885
2886 private void writeLogMessageMissed() {
2887 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2888 this.writeMessage();
2889 }
2890
2891 private void writeMessage() {
2892 final Conversational conversational = message.getConversation();
2893 if (conversational instanceof Conversation) {
2894 ((Conversation) conversational).add(this.message);
2895 xmppConnectionService.createMessageAsync(message);
2896 xmppConnectionService.updateConversationUi();
2897 } else {
2898 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2899 }
2900 }
2901
2902 public Optional<VideoTrack> getLocalVideoTrack() {
2903 return webRTCWrapper.getLocalVideoTrack();
2904 }
2905
2906 public Optional<VideoTrack> getRemoteVideoTrack() {
2907 return webRTCWrapper.getRemoteVideoTrack();
2908 }
2909
2910 public EglBase.Context getEglBaseContext() {
2911 return webRTCWrapper.getEglBaseContext();
2912 }
2913
2914 void setProposedMedia(final Set<Media> media) {
2915 this.proposedMedia = media;
2916 this.callIntegration.setVideoState(
2917 Media.audioOnly(media)
2918 ? VideoProfile.STATE_AUDIO_ONLY
2919 : VideoProfile.STATE_BIDIRECTIONAL);
2920 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2921 }
2922
2923 public void fireStateUpdate() {
2924 final RtpEndUserState endUserState = getEndUserState();
2925 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2926 id.account, id.with, id.sessionId, endUserState);
2927 }
2928
2929 public boolean isSwitchToVideoAvailable() {
2930 final boolean prerequisite =
2931 Media.audioOnly(getMedia())
2932 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2933 .contains(getEndUserState());
2934 return prerequisite && remoteHasVideoFeature();
2935 }
2936
2937 private boolean remoteHasVideoFeature() {
2938 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2939 }
2940
2941 private boolean remoteHasSdpOfferAnswer() {
2942 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2943 }
2944
2945 @Override
2946 public Account getAccount() {
2947 return id.account;
2948 }
2949
2950 @Override
2951 public Jid getWith() {
2952 return id.with;
2953 }
2954
2955 @Override
2956 public String getSessionId() {
2957 return id.sessionId;
2958 }
2959
2960 private interface OnIceServersDiscovered {
2961 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2962 }
2963}