1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50
51import im.conversations.android.xmpp.model.jingle.Jingle;
52import im.conversations.android.xmpp.model.stanza.Iq;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROPOSED,
78 State.PROCEED,
79 State.SESSION_INITIALIZED_PRE_APPROVED,
80 State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82
83 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
84 private final Queue<
85 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
86 pendingIceCandidates = new LinkedList<>();
87 private final OmemoVerification omemoVerification = new OmemoVerification();
88 private final CallIntegration callIntegration;
89 private final Message message;
90
91 private Set<Media> proposedMedia;
92 private RtpContentMap initiatorRtpContentMap;
93 private RtpContentMap responderRtpContentMap;
94 private RtpContentMap incomingContentAdd;
95 private RtpContentMap outgoingContentAdd;
96 private IceUdpTransportInfo.Setup peerDtlsSetup;
97 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
98 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
99 private ScheduledFuture<?> ringingTimeoutFuture;
100
101 JingleRtpConnection(
102 final JingleConnectionManager jingleConnectionManager,
103 final Id id,
104 final Jid initiator) {
105 this(
106 jingleConnectionManager,
107 id,
108 initiator,
109 new CallIntegration(
110 jingleConnectionManager
111 .getXmppConnectionService()
112 .getApplicationContext()));
113 this.callIntegration.setAddress(
114 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
115 final var contact = id.getContact();
116 this.callIntegration.setCallerDisplayName(
117 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
118 this.callIntegration.setInitialized();
119 }
120
121 JingleRtpConnection(
122 final JingleConnectionManager jingleConnectionManager,
123 final Id id,
124 final Jid initiator,
125 final CallIntegration callIntegration) {
126 super(jingleConnectionManager, id, initiator);
127 final Conversation conversation =
128 jingleConnectionManager
129 .getXmppConnectionService()
130 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
131 this.message =
132 new Message(
133 conversation,
134 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
135 Message.TYPE_RTP_SESSION,
136 id.sessionId);
137 this.callIntegration = callIntegration;
138 this.callIntegration.setCallback(this);
139 }
140
141 @Override
142 synchronized void deliverPacket(final Iq iq) {
143 final var jingle = iq.getExtension(Jingle.class);
144 switch (jingle.getAction()) {
145 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
146 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
147 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
148 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
149 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
150 case CONTENT_ACCEPT -> receiveContentAccept(iq);
151 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
152 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
153 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
154 default -> {
155 respondOk(iq);
156 Log.d(
157 Config.LOGTAG,
158 String.format(
159 "%s: received unhandled jingle action %s",
160 id.account.getJid().asBareJid(), jingle.getAction()));
161 }
162 }
163 }
164
165 @Override
166 synchronized void notifyRebound() {
167 if (isTerminated()) {
168 return;
169 }
170 webRTCWrapper.close();
171 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
172 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
173 }
174 if (isInState(
175 State.SESSION_INITIALIZED,
176 State.SESSION_INITIALIZED_PRE_APPROVED,
177 State.SESSION_ACCEPTED)) {
178 // we might have already changed resources (full jid) at this point; so this might not
179 // even reach the other party
180 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
181 } else {
182 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
183 finish();
184 }
185 }
186
187 private void receiveSessionTerminate(final Iq jinglePacket) {
188 respondOk(jinglePacket);
189 final var jingle = jinglePacket.getExtension(Jingle.class);
190 final Jingle.ReasonWrapper wrapper = jingle.getReason();
191 final State previous = this.state;
192 Log.d(
193 Config.LOGTAG,
194 id.account.getJid().asBareJid()
195 + ": received session terminate reason="
196 + wrapper.reason
197 + "("
198 + Strings.nullToEmpty(wrapper.text)
199 + ") while in state "
200 + previous);
201 if (TERMINATED.contains(previous)) {
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": ignoring session terminate because already in "
206 + previous);
207 return;
208 }
209 webRTCWrapper.close();
210 final State target = reasonToState(wrapper.reason);
211 transitionOrThrow(target);
212 writeLogMessage(target);
213 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
214 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
215 }
216 finish();
217 }
218
219 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
220 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
221 // INITIALIZED only after transport-info has been received
222 if (isInState(
223 State.NULL,
224 State.PROCEED,
225 State.SESSION_INITIALIZED,
226 State.SESSION_INITIALIZED_PRE_APPROVED,
227 State.SESSION_ACCEPTED)) {
228 final RtpContentMap contentMap;
229 try {
230 contentMap = RtpContentMap.of(jingle);
231 } catch (final IllegalArgumentException | NullPointerException e) {
232 Log.d(
233 Config.LOGTAG,
234 id.account.getJid().asBareJid()
235 + ": improperly formatted contents; ignoring",
236 e);
237 respondOk(jinglePacket);
238 return;
239 }
240 receiveTransportInfo(jinglePacket, contentMap);
241 } else {
242 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
243 }
244 }
245
246 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
247 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
248 candidates = contentMap.contents.entrySet();
249 final RtpContentMap remote = getRemoteContentMap();
250 final Set<String> remoteContentIds =
251 remote == null ? Collections.emptySet() : remote.contents.keySet();
252 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
253 Log.d(
254 Config.LOGTAG,
255 "received transport-info for unknown contents "
256 + contentMap.contents.keySet()
257 + " (known: "
258 + remoteContentIds
259 + ")");
260 respondOk(jinglePacket);
261 pendingIceCandidates.addAll(candidates);
262 return;
263 }
264 if (this.state != State.SESSION_ACCEPTED) {
265 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
266 respondOk(jinglePacket);
267 pendingIceCandidates.addAll(candidates);
268 return;
269 }
270 // zero candidates + modified credentials are an ICE restart offer
271 if (checkForIceRestart(jinglePacket, contentMap)) {
272 return;
273 }
274 respondOk(jinglePacket);
275 try {
276 processCandidates(candidates);
277 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
278 Log.w(
279 Config.LOGTAG,
280 id.account.getJid().asBareJid()
281 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
282 }
283 }
284
285 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
286 final RtpContentMap modification;
287 try {
288 modification = RtpContentMap.of(jingle);
289 modification.requireContentDescriptions();
290 } catch (final RuntimeException e) {
291 Log.d(
292 Config.LOGTAG,
293 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
294 Throwables.getRootCause(e));
295 respondOk(iq);
296 webRTCWrapper.close();
297 sendSessionTerminate(Reason.of(e), e.getMessage());
298 return;
299 }
300 if (isInState(State.SESSION_ACCEPTED)) {
301 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
302 final ListenableFuture<RtpContentMap> future =
303 receiveRtpContentMap(
304 modification,
305 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
306 Futures.addCallback(
307 future,
308 new FutureCallback<>() {
309 @Override
310 public void onSuccess(final RtpContentMap rtpContentMap) {
311 receiveContentAdd(iq, rtpContentMap);
312 }
313
314 @Override
315 public void onFailure(@NonNull Throwable throwable) {
316 respondOk(iq);
317 final Throwable rootCause = Throwables.getRootCause(throwable);
318 Log.d(
319 Config.LOGTAG,
320 id.account.getJid().asBareJid()
321 + ": improperly formatted contents in content-add",
322 throwable);
323 webRTCWrapper.close();
324 sendSessionTerminate(
325 Reason.ofThrowable(rootCause), rootCause.getMessage());
326 }
327 },
328 MoreExecutors.directExecutor());
329 } else {
330 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
331 }
332 }
333
334 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
335 final RtpContentMap remote = getRemoteContentMap();
336 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
337 respondOk(jinglePacket);
338 this.webRTCWrapper.close();
339 sendSessionTerminate(
340 Reason.FAILED_APPLICATION,
341 String.format(
342 "contents with names %s already exists",
343 Joiner.on(", ").join(modification.getNames())));
344 return;
345 }
346 final ContentAddition contentAddition =
347 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
348
349 final RtpContentMap outgoing = this.outgoingContentAdd;
350 final Set<ContentAddition.Summary> outgoingContentAddSummary =
351 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
352
353 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
354 if (isInitiator()) {
355 Log.d(
356 Config.LOGTAG,
357 id.getAccount().getJid().asBareJid()
358 + ": respond with tie break to matching content-add offer");
359 respondWithTieBreak(jinglePacket);
360 } else {
361 Log.d(
362 Config.LOGTAG,
363 id.getAccount().getJid().asBareJid()
364 + ": automatically accept matching content-add offer");
365 acceptContentAdd(contentAddition.summary, modification);
366 }
367 return;
368 }
369
370 // once we can display multiple video tracks we can be more loose with this condition
371 // theoretically it should also be fine to automatically accept audio only contents
372 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
373 Log.d(
374 Config.LOGTAG,
375 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
376 this.incomingContentAdd = modification;
377 respondOk(jinglePacket);
378 updateEndUserState();
379 } else {
380 respondOk(jinglePacket);
381 // TODO do we want to add a reason?
382 rejectContentAdd(modification);
383 }
384 }
385
386 private void receiveContentAccept(final Iq jinglePacket) {
387 final var jingle = jinglePacket.getExtension(Jingle.class);
388 final RtpContentMap receivedContentAccept;
389 try {
390 receivedContentAccept = RtpContentMap.of(jingle);
391 receivedContentAccept.requireContentDescriptions();
392 } catch (final RuntimeException e) {
393 Log.d(
394 Config.LOGTAG,
395 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
396 Throwables.getRootCause(e));
397 respondOk(jinglePacket);
398 webRTCWrapper.close();
399 sendSessionTerminate(Reason.of(e), e.getMessage());
400 return;
401 }
402
403 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
404 if (outgoingContentAdd == null) {
405 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
406 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
407 return;
408 }
409 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
410 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
411 this.outgoingContentAdd = null;
412 respondOk(jinglePacket);
413 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
414 final ListenableFuture<RtpContentMap> future =
415 receiveRtpContentMap(
416 receivedContentAccept,
417 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
418 Futures.addCallback(
419 future,
420 new FutureCallback<>() {
421 @Override
422 public void onSuccess(final RtpContentMap result) {
423 receiveContentAccept(result);
424 }
425
426 @Override
427 public void onFailure(@NonNull final Throwable throwable) {
428 webRTCWrapper.close();
429 sendSessionTerminate(
430 Reason.ofThrowable(throwable), throwable.getMessage());
431 }
432 },
433 MoreExecutors.directExecutor());
434 } else {
435 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
436 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
437 }
438 }
439
440 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
441 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
442 final RtpContentMap modifiedContentMap =
443 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
444
445 setRemoteContentMap(modifiedContentMap);
446
447 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
448
449 final org.webrtc.SessionDescription sdp =
450 new org.webrtc.SessionDescription(
451 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
452
453 try {
454 this.webRTCWrapper.setRemoteDescription(sdp).get();
455 } catch (final Exception e) {
456 final Throwable cause = Throwables.getRootCause(e);
457 Log.d(
458 Config.LOGTAG,
459 id.getAccount().getJid().asBareJid()
460 + ": unable to set remote description after receiving content-accept",
461 cause);
462 webRTCWrapper.close();
463 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
464 return;
465 }
466 Log.d(
467 Config.LOGTAG,
468 id.getAccount().getJid().asBareJid()
469 + ": remote has accepted content-add "
470 + ContentAddition.summary(receivedContentAccept));
471 processCandidates(receivedContentAccept.contents.entrySet());
472 updateEndUserState();
473 }
474
475 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
476 if (this.state != State.SESSION_ACCEPTED) {
477 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
478 return;
479 }
480 final Map<String, Content.Senders> modification =
481 Maps.transformEntries(
482 jingle.getJingleContents(), (key, value) -> value.getSenders());
483 final boolean isInitiator = isInitiator();
484 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
485 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
486 final Set<String> currentOutgoingMediaIds =
487 currentOutgoing == null
488 ? Collections.emptySet()
489 : currentOutgoing.contents.keySet();
490 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
491 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
492 respondOk(jinglePacket);
493 final RtpContentMap modifiedContentMap;
494 try {
495 modifiedContentMap =
496 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
497 } catch (final IllegalArgumentException e) {
498 webRTCWrapper.close();
499 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
500 return;
501 }
502 this.outgoingContentAdd = modifiedContentMap;
503 Log.d(
504 Config.LOGTAG,
505 id.account.getJid().asBareJid()
506 + ": processed content-modification for pending content-add");
507 } else if (remoteContentMap != null
508 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
509 respondOk(jinglePacket);
510 final RtpContentMap modifiedRemoteContentMap;
511 try {
512 modifiedRemoteContentMap =
513 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
514 } catch (final IllegalArgumentException e) {
515 webRTCWrapper.close();
516 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
517 return;
518 }
519 final SessionDescription offer;
520 try {
521 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
522 } catch (final IllegalArgumentException | NullPointerException e) {
523 Log.d(
524 Config.LOGTAG,
525 id.getAccount().getJid().asBareJid()
526 + ": unable convert offer from content-modify to SDP",
527 e);
528 webRTCWrapper.close();
529 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
530 return;
531 }
532 Log.d(
533 Config.LOGTAG,
534 id.account.getJid().asBareJid() + ": auto accepting content-modification");
535 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
536 } else {
537 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
538 respondWithItemNotFound(jinglePacket);
539 }
540 }
541
542 private void autoAcceptContentModify(
543 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
544 this.setRemoteContentMap(modifiedRemoteContentMap);
545 final org.webrtc.SessionDescription sdp =
546 new org.webrtc.SessionDescription(
547 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
548 try {
549 this.webRTCWrapper.setRemoteDescription(sdp).get();
550 // auto accept is only done when we already have tracks
551 final SessionDescription answer = setLocalSessionDescription();
552 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
553 modifyLocalContentMap(rtpContentMap);
554 // we do not need to send an answer but do we have to resend the candidates currently in
555 // SDP?
556 // resendCandidatesFromSdp(answer);
557 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
558 } catch (final Exception e) {
559 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
560 webRTCWrapper.close();
561 sendSessionTerminate(Reason.FAILED_APPLICATION);
562 }
563 }
564
565 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
566 final SessionDescription answer) {
567 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
568 new ImmutableMultimap.Builder<>();
569 for (final SessionDescription.Media media : answer.media) {
570 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
571 if (Strings.isNullOrEmpty(mid)) {
572 continue;
573 }
574 for (final String sdpCandidate : media.attributes.get("candidate")) {
575 final IceUdpTransportInfo.Candidate candidate =
576 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
577 if (candidate != null) {
578 candidateBuilder.put(mid, candidate);
579 }
580 }
581 }
582 return candidateBuilder.build();
583 }
584
585 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
586 final RtpContentMap receivedContentReject;
587 try {
588 receivedContentReject = RtpContentMap.of(jingle);
589 } catch (final RuntimeException e) {
590 Log.d(
591 Config.LOGTAG,
592 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
593 Throwables.getRootCause(e));
594 respondOk(jinglePacket);
595 this.webRTCWrapper.close();
596 sendSessionTerminate(Reason.of(e), e.getMessage());
597 return;
598 }
599
600 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
601 if (outgoingContentAdd == null) {
602 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
603 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
604 return;
605 }
606 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
607 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
608 this.outgoingContentAdd = null;
609 respondOk(jinglePacket);
610 Log.d(Config.LOGTAG, jinglePacket.toString());
611 receiveContentReject(ourSummary);
612 } else {
613 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
614 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
615 }
616 }
617
618 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
619 try {
620 this.webRTCWrapper.removeTrack(Media.VIDEO);
621 final RtpContentMap localContentMap = customRollback();
622 modifyLocalContentMap(localContentMap);
623 } catch (final Exception e) {
624 final Throwable cause = Throwables.getRootCause(e);
625 Log.d(
626 Config.LOGTAG,
627 id.getAccount().getJid().asBareJid()
628 + ": unable to rollback local description after receiving content-reject",
629 cause);
630 webRTCWrapper.close();
631 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
632 return;
633 }
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid().asBareJid()
637 + ": remote has rejected our content-add "
638 + summary);
639 }
640
641 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
642 final RtpContentMap receivedContentRemove;
643 try {
644 receivedContentRemove = RtpContentMap.of(jingle);
645 receivedContentRemove.requireContentDescriptions();
646 } catch (final RuntimeException e) {
647 Log.d(
648 Config.LOGTAG,
649 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
650 Throwables.getRootCause(e));
651 respondOk(jinglePacket);
652 this.webRTCWrapper.close();
653 sendSessionTerminate(Reason.of(e), e.getMessage());
654 return;
655 }
656 respondOk(jinglePacket);
657 receiveContentRemove(receivedContentRemove);
658 }
659
660 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
661 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
662 final Set<ContentAddition.Summary> contentAddSummary =
663 incomingContentAdd == null
664 ? Collections.emptySet()
665 : ContentAddition.summary(incomingContentAdd);
666 final Set<ContentAddition.Summary> removeSummary =
667 ContentAddition.summary(receivedContentRemove);
668 if (contentAddSummary.equals(removeSummary)) {
669 this.incomingContentAdd = null;
670 updateEndUserState();
671 } else {
672 webRTCWrapper.close();
673 sendSessionTerminate(
674 Reason.FAILED_APPLICATION,
675 String.format(
676 "%s only supports %s as a means to retract a not yet accepted %s",
677 BuildConfig.APP_NAME,
678 Jingle.Action.CONTENT_REMOVE,
679 Jingle.Action.CONTENT_ADD));
680 }
681 }
682
683 public synchronized void retractContentAdd() {
684 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
685 if (outgoingContentAdd == null) {
686 throw new IllegalStateException("Not outgoing content add");
687 }
688 try {
689 webRTCWrapper.removeTrack(Media.VIDEO);
690 final RtpContentMap localContentMap = customRollback();
691 modifyLocalContentMap(localContentMap);
692 } catch (final Exception e) {
693 final Throwable cause = Throwables.getRootCause(e);
694 Log.d(
695 Config.LOGTAG,
696 id.getAccount().getJid().asBareJid()
697 + ": unable to rollback local description after trying to retract content-add",
698 cause);
699 webRTCWrapper.close();
700 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
701 return;
702 }
703 this.outgoingContentAdd = null;
704 final Iq retract =
705 outgoingContentAdd
706 .toStub()
707 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
708 this.send(retract);
709 Log.d(
710 Config.LOGTAG,
711 id.getAccount().getJid()
712 + ": retract content-add "
713 + ContentAddition.summary(outgoingContentAdd));
714 }
715
716 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
717 final SessionDescription sdp = setLocalSessionDescription();
718 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
719 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
720 this.webRTCWrapper
721 .setRemoteDescription(
722 new org.webrtc.SessionDescription(
723 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
724 .get();
725 return localRtpContentMap;
726 }
727
728 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
729 final RtpContentMap currentRemote = getRemoteContentMap();
730 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
731 if (diff.isEmpty()) {
732 throw new IllegalStateException(
733 "Unexpected rollback condition. No difference between local and remote");
734 }
735 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
736 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
737 final RtpContentMap nextRemote =
738 currentRemote.addContent(
739 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
740 return SessionDescription.of(nextRemote, isResponder());
741 }
742 throw new IllegalStateException(
743 "Unexpected rollback condition. Senders were not uniformly none");
744 }
745
746 public synchronized void acceptContentAdd(
747 @NonNull final Set<ContentAddition.Summary> contentAddition) {
748 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
749 if (incomingContentAdd == null) {
750 throw new IllegalStateException("No incoming content add");
751 }
752
753 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
754 this.incomingContentAdd = null;
755 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
756 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
757 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
758 Log.d(
759 Config.LOGTAG,
760 "content addition is receive only. we want to upgrade to 'both'");
761 final RtpContentMap modifiedSenders =
762 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
763 final Iq proposedContentModification =
764 modifiedSenders
765 .toStub()
766 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
767 proposedContentModification.setTo(id.with);
768 xmppConnectionService.sendIqPacket(
769 id.account,
770 proposedContentModification,
771 (response) -> {
772 if (response.getType() == Iq.Type.RESULT) {
773 Log.d(
774 Config.LOGTAG,
775 id.account.getJid().asBareJid()
776 + ": remote has accepted our upgrade to senders=both");
777 acceptContentAdd(
778 ContentAddition.summary(modifiedSenders), modifiedSenders);
779 } else {
780 Log.d(
781 Config.LOGTAG,
782 id.account.getJid().asBareJid()
783 + ": remote has rejected our upgrade to senders=both");
784 acceptContentAdd(contentAddition, incomingContentAdd);
785 }
786 });
787 } else {
788 acceptContentAdd(contentAddition, incomingContentAdd);
789 }
790 } else {
791 throw new IllegalStateException(
792 "Accepted content add does not match pending content-add");
793 }
794 }
795
796 private void acceptContentAdd(
797 @NonNull final Set<ContentAddition.Summary> contentAddition,
798 final RtpContentMap incomingContentAdd) {
799 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
800 final RtpContentMap modifiedContentMap =
801 getRemoteContentMap().addContent(incomingContentAdd, setup);
802 this.setRemoteContentMap(modifiedContentMap);
803
804 final SessionDescription offer;
805 try {
806 offer = SessionDescription.of(modifiedContentMap, isResponder());
807 } catch (final IllegalArgumentException | NullPointerException e) {
808 Log.d(
809 Config.LOGTAG,
810 id.getAccount().getJid().asBareJid()
811 + ": unable convert offer from content-add to SDP",
812 e);
813 webRTCWrapper.close();
814 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
815 return;
816 }
817 this.incomingContentAdd = null;
818 acceptContentAdd(contentAddition, offer);
819 }
820
821 private void acceptContentAdd(
822 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
823 final org.webrtc.SessionDescription sdp =
824 new org.webrtc.SessionDescription(
825 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
826 try {
827 this.webRTCWrapper.setRemoteDescription(sdp).get();
828
829 // TODO add tracks for 'media' where contentAddition.senders matches
830
831 // TODO if senders.sending(isInitiator())
832
833 this.webRTCWrapper.addTrack(Media.VIDEO);
834
835 // TODO add additional transceivers for recv only cases
836
837 final SessionDescription answer = setLocalSessionDescription();
838 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
839
840 final RtpContentMap contentAcceptMap =
841 rtpContentMap.toContentModification(
842 Collections2.transform(contentAddition, ca -> ca.name));
843
844 Log.d(
845 Config.LOGTAG,
846 id.getAccount().getJid().asBareJid()
847 + ": sending content-accept "
848 + ContentAddition.summary(contentAcceptMap));
849
850 addIceCandidatesFromBlackLog();
851
852 modifyLocalContentMap(rtpContentMap);
853 final ListenableFuture<RtpContentMap> future =
854 prepareOutgoingContentMap(contentAcceptMap);
855 Futures.addCallback(
856 future,
857 new FutureCallback<>() {
858 @Override
859 public void onSuccess(final RtpContentMap rtpContentMap) {
860 sendContentAccept(rtpContentMap);
861 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
862 }
863
864 @Override
865 public void onFailure(@NonNull final Throwable throwable) {
866 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
867 }
868 },
869 MoreExecutors.directExecutor());
870 } catch (final Exception e) {
871 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
872 webRTCWrapper.close();
873 sendSessionTerminate(Reason.FAILED_APPLICATION);
874 }
875 }
876
877 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
878 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
879 send(iq);
880 }
881
882 public synchronized void rejectContentAdd() {
883 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
884 if (incomingContentAdd == null) {
885 throw new IllegalStateException("No incoming content add");
886 }
887 this.incomingContentAdd = null;
888 updateEndUserState();
889 rejectContentAdd(incomingContentAdd);
890 }
891
892 private void rejectContentAdd(final RtpContentMap contentMap) {
893 final Iq iq =
894 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
895 Log.d(
896 Config.LOGTAG,
897 id.getAccount().getJid().asBareJid()
898 + ": rejecting content "
899 + ContentAddition.summary(contentMap));
900 send(iq);
901 }
902
903 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
904 final RtpContentMap existing = getRemoteContentMap();
905 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
906 final IceUdpTransportInfo.Credentials newCredentials;
907 try {
908 existingCredentials = existing.getCredentials();
909 newCredentials = rtpContentMap.getDistinctCredentials();
910 } catch (final IllegalStateException e) {
911 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
912 return false;
913 }
914 if (existingCredentials.contains(newCredentials)) {
915 return false;
916 }
917 // TODO an alternative approach is to check if we already got an iq result to our
918 // ICE-restart
919 // and if that's the case we are seeing an answer.
920 // This might be more spec compliant but also more error prone potentially
921 final boolean isSignalStateStable =
922 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
923 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
924 final boolean isOffer = rtpContentMap.emptyCandidates();
925 final RtpContentMap restartContentMap;
926 try {
927 if (isOffer) {
928 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
929 restartContentMap =
930 existing.modifiedCredentials(
931 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
932 } else {
933 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
934 Log.d(
935 Config.LOGTAG,
936 "received confirmation of ICE restart"
937 + newCredentials
938 + " peer_setup="
939 + setup);
940 // DTLS setup attribute needs to be rewritten to reflect current peer state
941 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
942 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
943 }
944 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
945 return isOffer;
946 } else {
947 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
948 respondWithTieBreak(jinglePacket);
949 return true;
950 }
951 } catch (final Exception exception) {
952 respondOk(jinglePacket);
953 final Throwable rootCause = Throwables.getRootCause(exception);
954 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
955 // If this happens a termination is already in progress
956 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
957 return true;
958 }
959 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
960 webRTCWrapper.close();
961 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
962 return true;
963 }
964 }
965
966 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
967 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
968 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
969 throw new IllegalStateException("Invalid peer setup");
970 }
971 return peerSetup;
972 }
973
974 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
975 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
976 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
977 }
978 this.peerDtlsSetup = setup;
979 }
980
981 private boolean applyIceRestart(
982 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
983 throws ExecutionException, InterruptedException {
984 final SessionDescription sessionDescription =
985 SessionDescription.of(restartContentMap, isResponder());
986 final org.webrtc.SessionDescription.Type type =
987 isOffer
988 ? org.webrtc.SessionDescription.Type.OFFER
989 : org.webrtc.SessionDescription.Type.ANSWER;
990 org.webrtc.SessionDescription sdp =
991 new org.webrtc.SessionDescription(type, sessionDescription.toString());
992 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
993 if (isInitiator()) {
994 // We ignore the offer and respond with tie-break. This will clause the responder
995 // not to apply the content map
996 return false;
997 }
998 }
999 webRTCWrapper.setRemoteDescription(sdp).get();
1000 setRemoteContentMap(restartContentMap);
1001 if (isOffer) {
1002 final SessionDescription localSessionDescription = setLocalSessionDescription();
1003 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1004 // We need to respond OK before sending any candidates
1005 respondOk(jinglePacket);
1006 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1007 } else {
1008 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1009 }
1010 return true;
1011 }
1012
1013 private void processCandidates(
1014 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1015 contents) {
1016 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1017 content : contents) {
1018 processCandidate(content);
1019 }
1020 }
1021
1022 private void processCandidate(
1023 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1024 content) {
1025 final RtpContentMap rtpContentMap = getRemoteContentMap();
1026 final List<String> indices = toIdentificationTags(rtpContentMap);
1027 final String sdpMid = content.getKey(); // aka content name
1028 final IceUdpTransportInfo transport = content.getValue().transport;
1029 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1030
1031 // TODO check that credentials remained the same
1032
1033 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1034 final String sdp;
1035 try {
1036 sdp = candidate.toSdpAttribute(credentials.ufrag);
1037 } catch (final IllegalArgumentException e) {
1038 Log.d(
1039 Config.LOGTAG,
1040 id.account.getJid().asBareJid()
1041 + ": ignoring invalid ICE candidate "
1042 + e.getMessage());
1043 continue;
1044 }
1045 final int mLineIndex = indices.indexOf(sdpMid);
1046 if (mLineIndex < 0) {
1047 Log.w(
1048 Config.LOGTAG,
1049 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1050 }
1051 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1052 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1053 this.webRTCWrapper.addIceCandidate(iceCandidate);
1054 }
1055 }
1056
1057 private RtpContentMap getRemoteContentMap() {
1058 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1059 }
1060
1061 private RtpContentMap getLocalContentMap() {
1062 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1063 }
1064
1065 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1066 final Group originalGroup = rtpContentMap.group;
1067 final List<String> identificationTags =
1068 originalGroup == null
1069 ? rtpContentMap.getNames()
1070 : originalGroup.getIdentificationTags();
1071 if (identificationTags.size() == 0) {
1072 Log.w(
1073 Config.LOGTAG,
1074 id.account.getJid().asBareJid()
1075 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1076 }
1077 return identificationTags;
1078 }
1079
1080 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1081 final Jingle jinglePacket, final boolean expectVerification) {
1082 try {
1083 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1084 } catch (final Exception e) {
1085 return Futures.immediateFailedFuture(e);
1086 }
1087 }
1088
1089 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1090 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1091 Log.d(
1092 Config.LOGTAG,
1093 "receiveRtpContentMap("
1094 + receivedContentMap.getClass().getSimpleName()
1095 + ",expectVerification="
1096 + expectVerification
1097 + ")");
1098 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1099 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1100 id.account
1101 .getAxolotlService()
1102 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1103 return Futures.transform(
1104 future,
1105 omemoVerifiedPayload -> {
1106 // TODO test if an exception here triggers a correct abort
1107 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1108 Log.d(
1109 Config.LOGTAG,
1110 id.account.getJid().asBareJid()
1111 + ": received verifiable DTLS fingerprint via "
1112 + omemoVerification);
1113 return omemoVerifiedPayload.getPayload();
1114 },
1115 MoreExecutors.directExecutor());
1116 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1117 return Futures.immediateFailedFuture(
1118 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1119 } else {
1120 return Futures.immediateFuture(receivedContentMap);
1121 }
1122 }
1123
1124 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1125 if (isInitiator()) {
1126 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1127 return;
1128 }
1129 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1130 Futures.addCallback(
1131 future,
1132 new FutureCallback<>() {
1133 @Override
1134 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1135 receiveSessionInitiate(jinglePacket, rtpContentMap);
1136 }
1137
1138 @Override
1139 public void onFailure(@NonNull final Throwable throwable) {
1140 respondOk(jinglePacket);
1141 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1142 }
1143 },
1144 MoreExecutors.directExecutor());
1145 }
1146
1147 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1148 try {
1149 contentMap.requireContentDescriptions();
1150 contentMap.requireDTLSFingerprint(true);
1151 } catch (final RuntimeException e) {
1152 Log.d(
1153 Config.LOGTAG,
1154 id.account.getJid().asBareJid() + ": improperly formatted contents",
1155 Throwables.getRootCause(e));
1156 respondOk(jinglePacket);
1157 sendSessionTerminate(Reason.of(e), e.getMessage());
1158 return;
1159 }
1160 Log.d(
1161 Config.LOGTAG,
1162 "processing session-init with " + contentMap.contents.size() + " contents");
1163 final State target;
1164 if (this.state == State.PROCEED) {
1165 Preconditions.checkState(
1166 proposedMedia != null && proposedMedia.size() > 0,
1167 "proposed media must be set when processing pre-approved session-initiate");
1168 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1169 sendSessionTerminate(
1170 Reason.SECURITY_ERROR,
1171 String.format(
1172 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1173 this.proposedMedia, contentMap.getMedia()));
1174 return;
1175 }
1176 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1177 } else {
1178 target = State.SESSION_INITIALIZED;
1179 setProposedMedia(contentMap.getMedia());
1180 }
1181 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1182 respondOk(jinglePacket);
1183 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1184 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1185 Log.d(
1186 Config.LOGTAG,
1187 id.account.getJid().asBareJid()
1188 + ": automatically accepting session-initiate");
1189 sendSessionAccept();
1190 } else {
1191 Log.d(
1192 Config.LOGTAG,
1193 id.account.getJid().asBareJid()
1194 + ": received not pre-approved session-initiate. start ringing");
1195 startRinging();
1196 }
1197 } else {
1198 Log.d(
1199 Config.LOGTAG,
1200 String.format(
1201 "%s: received session-initiate while in state %s",
1202 id.account.getJid().asBareJid(), state));
1203 terminateWithOutOfOrder(jinglePacket);
1204 }
1205 }
1206
1207 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1208 if (isResponder()) {
1209 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1210 return;
1211 }
1212 final ListenableFuture<RtpContentMap> future =
1213 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1214 Futures.addCallback(
1215 future,
1216 new FutureCallback<>() {
1217 @Override
1218 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1219 receiveSessionAccept(jinglePacket, rtpContentMap);
1220 }
1221
1222 @Override
1223 public void onFailure(@NonNull final Throwable throwable) {
1224 respondOk(jinglePacket);
1225 Log.d(
1226 Config.LOGTAG,
1227 id.account.getJid().asBareJid()
1228 + ": improperly formatted contents in session-accept",
1229 throwable);
1230 webRTCWrapper.close();
1231 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1232 }
1233 },
1234 MoreExecutors.directExecutor());
1235 }
1236
1237 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1238 try {
1239 contentMap.requireContentDescriptions();
1240 contentMap.requireDTLSFingerprint();
1241 } catch (final RuntimeException e) {
1242 respondOk(jinglePacket);
1243 Log.d(
1244 Config.LOGTAG,
1245 id.account.getJid().asBareJid()
1246 + ": improperly formatted contents in session-accept",
1247 e);
1248 webRTCWrapper.close();
1249 sendSessionTerminate(Reason.of(e), e.getMessage());
1250 return;
1251 }
1252 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1253 if (!initiatorMedia.equals(contentMap.getMedia())) {
1254 sendSessionTerminate(
1255 Reason.SECURITY_ERROR,
1256 String.format(
1257 "Your session-included included media %s but our session-initiate was %s",
1258 this.proposedMedia, contentMap.getMedia()));
1259 return;
1260 }
1261 Log.d(
1262 Config.LOGTAG,
1263 "processing session-accept with " + contentMap.contents.size() + " contents");
1264 if (transition(State.SESSION_ACCEPTED)) {
1265 respondOk(jinglePacket);
1266 receiveSessionAccept(contentMap);
1267 } else {
1268 Log.d(
1269 Config.LOGTAG,
1270 String.format(
1271 "%s: received session-accept while in state %s",
1272 id.account.getJid().asBareJid(), state));
1273 respondOk(jinglePacket);
1274 }
1275 }
1276
1277 private void receiveSessionAccept(final RtpContentMap contentMap) {
1278 this.responderRtpContentMap = contentMap;
1279 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1280 final SessionDescription sessionDescription;
1281 try {
1282 sessionDescription = SessionDescription.of(contentMap, false);
1283 } catch (final IllegalArgumentException | NullPointerException e) {
1284 Log.d(
1285 Config.LOGTAG,
1286 id.account.getJid().asBareJid()
1287 + ": unable convert offer from session-accept to SDP",
1288 e);
1289 webRTCWrapper.close();
1290 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1291 return;
1292 }
1293 final org.webrtc.SessionDescription answer =
1294 new org.webrtc.SessionDescription(
1295 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1296 try {
1297 this.webRTCWrapper.setRemoteDescription(answer).get();
1298 } catch (final Exception e) {
1299 Log.d(
1300 Config.LOGTAG,
1301 id.account.getJid().asBareJid()
1302 + ": unable to set remote description after receiving session-accept",
1303 Throwables.getRootCause(e));
1304 webRTCWrapper.close();
1305 sendSessionTerminate(
1306 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1307 return;
1308 }
1309 processCandidates(contentMap.contents.entrySet());
1310 }
1311
1312 private void sendSessionAccept() {
1313 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1314 if (rtpContentMap == null) {
1315 throw new IllegalStateException("initiator RTP Content Map has not been set");
1316 }
1317 final SessionDescription offer;
1318 try {
1319 offer = SessionDescription.of(rtpContentMap, true);
1320 } catch (final IllegalArgumentException | NullPointerException e) {
1321 Log.d(
1322 Config.LOGTAG,
1323 id.account.getJid().asBareJid()
1324 + ": unable convert offer from session-initiate to SDP",
1325 e);
1326 webRTCWrapper.close();
1327 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1328 return;
1329 }
1330 sendSessionAccept(rtpContentMap.getMedia(), offer);
1331 }
1332
1333 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1334 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1335 }
1336
1337 private synchronized void sendSessionAccept(
1338 final Set<Media> media,
1339 final SessionDescription offer,
1340 final List<PeerConnection.IceServer> iceServers) {
1341 if (isTerminated()) {
1342 Log.w(
1343 Config.LOGTAG,
1344 id.account.getJid().asBareJid()
1345 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1346 return;
1347 }
1348 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1349 try {
1350 setupWebRTC(media, iceServers, !includeCandidates);
1351 } catch (final WebRTCWrapper.InitializationException e) {
1352 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1353 webRTCWrapper.close();
1354 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1355 return;
1356 }
1357 final org.webrtc.SessionDescription sdp =
1358 new org.webrtc.SessionDescription(
1359 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1360 try {
1361 this.webRTCWrapper.setRemoteDescription(sdp).get();
1362 addIceCandidatesFromBlackLog();
1363 org.webrtc.SessionDescription webRTCSessionDescription =
1364 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1365 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1366 } catch (final Exception e) {
1367 failureToAcceptSession(e);
1368 }
1369 }
1370
1371 private void failureToAcceptSession(final Throwable throwable) {
1372 if (isTerminated()) {
1373 return;
1374 }
1375 final Throwable rootCause = Throwables.getRootCause(throwable);
1376 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1377 webRTCWrapper.close();
1378 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1379 }
1380
1381 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1382 if (isTerminated()) {
1383 return;
1384 }
1385 final Throwable rootCause = Throwables.getRootCause(throwable);
1386 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1387 webRTCWrapper.close();
1388 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1389 }
1390
1391 private void addIceCandidatesFromBlackLog() {
1392 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1393 while ((foo = this.pendingIceCandidates.poll()) != null) {
1394 processCandidate(foo);
1395 Log.d(
1396 Config.LOGTAG,
1397 id.account.getJid().asBareJid() + ": added candidate from back log");
1398 }
1399 }
1400
1401 private void prepareSessionAccept(
1402 final org.webrtc.SessionDescription webRTCSessionDescription,
1403 final boolean includeCandidates) {
1404 final SessionDescription sessionDescription =
1405 SessionDescription.parse(webRTCSessionDescription.description);
1406 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1407 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1408 if (includeCandidates) {
1409 candidates = parseCandidates(sessionDescription);
1410 } else {
1411 candidates = ImmutableMultimap.of();
1412 }
1413 this.responderRtpContentMap = respondingRtpContentMap;
1414 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1415 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1416 prepareOutgoingContentMap(respondingRtpContentMap);
1417 Futures.addCallback(
1418 outgoingContentMapFuture,
1419 new FutureCallback<>() {
1420 @Override
1421 public void onSuccess(final RtpContentMap outgoingContentMap) {
1422 if (includeCandidates) {
1423 Log.d(
1424 Config.LOGTAG,
1425 "including "
1426 + candidates.size()
1427 + " candidates in session accept");
1428 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1429 } else {
1430 sendSessionAccept(outgoingContentMap);
1431 }
1432 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1433 }
1434
1435 @Override
1436 public void onFailure(@NonNull Throwable throwable) {
1437 failureToAcceptSession(throwable);
1438 }
1439 },
1440 MoreExecutors.directExecutor());
1441 }
1442
1443 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1444 if (isTerminated()) {
1445 Log.w(
1446 Config.LOGTAG,
1447 id.account.getJid().asBareJid()
1448 + ": preparing session accept was too slow. already terminated. nothing to do.");
1449 return;
1450 }
1451 transitionOrThrow(State.SESSION_ACCEPTED);
1452 final Iq sessionAccept =
1453 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1454 send(sessionAccept);
1455 }
1456
1457 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1458 final RtpContentMap rtpContentMap) {
1459 if (this.omemoVerification.hasDeviceId()) {
1460 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1461 verifiedPayloadFuture =
1462 id.account
1463 .getAxolotlService()
1464 .encrypt(
1465 rtpContentMap,
1466 id.with,
1467 omemoVerification.getDeviceId());
1468 return Futures.transform(
1469 verifiedPayloadFuture,
1470 verifiedPayload -> {
1471 omemoVerification.setOrEnsureEqual(verifiedPayload);
1472 return verifiedPayload.getPayload();
1473 },
1474 MoreExecutors.directExecutor());
1475 } else {
1476 return Futures.immediateFuture(rtpContentMap);
1477 }
1478 }
1479
1480 synchronized void deliveryMessage(
1481 final Jid from,
1482 final Element message,
1483 final String serverMessageId,
1484 final long timestamp) {
1485 Log.d(
1486 Config.LOGTAG,
1487 id.account.getJid().asBareJid()
1488 + ": delivered message to JingleRtpConnection "
1489 + message);
1490 switch (message.getName()) {
1491 case "propose" -> receivePropose(
1492 from, Propose.upgrade(message), serverMessageId, timestamp);
1493 case "proceed" -> receiveProceed(
1494 from, Proceed.upgrade(message), serverMessageId, timestamp);
1495 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1496 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1497 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1498 }
1499 }
1500
1501 void deliverFailedProceed(final String message) {
1502 Log.d(
1503 Config.LOGTAG,
1504 id.account.getJid().asBareJid()
1505 + ": receive message error for proceed message ("
1506 + Strings.nullToEmpty(message)
1507 + ")");
1508 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1509 webRTCWrapper.close();
1510 Log.d(
1511 Config.LOGTAG,
1512 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1513 this.finish();
1514 }
1515 }
1516
1517 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1518 final boolean originatedFromMyself =
1519 from.asBareJid().equals(id.account.getJid().asBareJid());
1520 if (originatedFromMyself) {
1521 if (transition(State.ACCEPTED)) {
1522 acceptedOnOtherDevice(serverMsgId, timestamp);
1523 } else {
1524 Log.d(
1525 Config.LOGTAG,
1526 id.account.getJid().asBareJid()
1527 + ": unable to transition to accept because already in state="
1528 + this.state);
1529 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1530 }
1531 } else {
1532 Log.d(
1533 Config.LOGTAG,
1534 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1535 }
1536 }
1537
1538 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1539 if (serverMsgId != null) {
1540 this.message.setServerMsgId(serverMsgId);
1541 }
1542 this.message.setTime(timestamp);
1543 this.message.setCarbon(true); // indicate that call was accepted on other device
1544 this.writeLogMessageSuccess(0);
1545 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1546 this.finish();
1547 }
1548
1549 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1550 final boolean originatedFromMyself =
1551 from.asBareJid().equals(id.account.getJid().asBareJid());
1552 // reject from another one of my clients
1553 if (originatedFromMyself) {
1554 receiveRejectFromMyself(serverMsgId, timestamp);
1555 } else if (isInitiator()) {
1556 if (from.equals(id.with)) {
1557 receiveRejectFromResponder();
1558 } else {
1559 Log.d(
1560 Config.LOGTAG,
1561 id.account.getJid()
1562 + ": ignoring reject from "
1563 + from
1564 + " for session with "
1565 + id.with);
1566 }
1567 } else {
1568 Log.d(
1569 Config.LOGTAG,
1570 id.account.getJid()
1571 + ": ignoring reject from "
1572 + from
1573 + " for session with "
1574 + id.with);
1575 }
1576 }
1577
1578 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1579 if (transition(State.REJECTED)) {
1580 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1581 this.finish();
1582 if (serverMsgId != null) {
1583 this.message.setServerMsgId(serverMsgId);
1584 }
1585 this.message.setTime(timestamp);
1586 this.message.setCarbon(true); // indicate that call was rejected on other device
1587 writeLogMessageMissed();
1588 } else {
1589 Log.d(
1590 Config.LOGTAG,
1591 "not able to transition into REJECTED because already in " + this.state);
1592 }
1593 }
1594
1595 private void receiveRejectFromResponder() {
1596 if (isInState(State.PROCEED)) {
1597 Log.d(
1598 Config.LOGTAG,
1599 id.account.getJid()
1600 + ": received reject while still in proceed. callee reconsidered");
1601 closeTransitionLogFinish(State.REJECTED_RACED);
1602 return;
1603 }
1604 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1605 Log.d(
1606 Config.LOGTAG,
1607 id.account.getJid()
1608 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1609 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1610 return;
1611 }
1612 Log.d(
1613 Config.LOGTAG,
1614 id.account.getJid()
1615 + ": ignoring reject from responder because already in state "
1616 + this.state);
1617 }
1618
1619 private void receivePropose(
1620 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1621 final boolean originatedFromMyself =
1622 from.asBareJid().equals(id.account.getJid().asBareJid());
1623 if (originatedFromMyself) {
1624 Log.d(
1625 Config.LOGTAG,
1626 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1627 } else if (transition(
1628 State.PROPOSED,
1629 () -> {
1630 final Collection<RtpDescription> descriptions =
1631 Collections2.transform(
1632 Collections2.filter(
1633 propose.getDescriptions(),
1634 d -> d instanceof RtpDescription),
1635 input -> (RtpDescription) input);
1636 final Collection<Media> media =
1637 Collections2.transform(descriptions, RtpDescription::getMedia);
1638 Preconditions.checkState(
1639 !media.contains(Media.UNKNOWN),
1640 "RTP descriptions contain unknown media");
1641 Log.d(
1642 Config.LOGTAG,
1643 id.account.getJid().asBareJid()
1644 + ": received session proposal from "
1645 + from
1646 + " for "
1647 + media);
1648 this.setProposedMedia(Sets.newHashSet(media));
1649 })) {
1650 if (serverMsgId != null) {
1651 this.message.setServerMsgId(serverMsgId);
1652 }
1653 this.message.setTime(timestamp);
1654 startRinging();
1655 // in environments where we always use discovery timeouts we always want to respond with
1656 // 'ringing'
1657 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1658 || id.getContact().showInContactList()) {
1659 sendJingleMessage("ringing");
1660 }
1661 } else {
1662 Log.d(
1663 Config.LOGTAG,
1664 id.account.getJid()
1665 + ": ignoring session proposal because already in "
1666 + state);
1667 }
1668 }
1669
1670 private void startRinging() {
1671 this.callIntegration.setRinging();
1672 Log.d(
1673 Config.LOGTAG,
1674 id.account.getJid().asBareJid()
1675 + ": received call from "
1676 + id.with
1677 + ". start ringing");
1678 ringingTimeoutFuture =
1679 jingleConnectionManager.schedule(
1680 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1681 if (CallIntegration.selfManaged(xmppConnectionService)) {
1682 return;
1683 }
1684 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1685 }
1686
1687 private synchronized void ringingTimeout() {
1688 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1689 switch (this.state) {
1690 case PROPOSED -> {
1691 message.markUnread();
1692 rejectCallFromProposed();
1693 }
1694 case SESSION_INITIALIZED -> {
1695 message.markUnread();
1696 rejectCallFromSessionInitiate();
1697 }
1698 }
1699 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1700 }
1701
1702 private void cancelRingingTimeout() {
1703 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1704 if (future != null && !future.isCancelled()) {
1705 future.cancel(false);
1706 }
1707 }
1708
1709 private void receiveProceed(
1710 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1711 final Set<Media> media =
1712 Preconditions.checkNotNull(
1713 this.proposedMedia, "Proposed media has to be set before handling proceed");
1714 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1715 if (from.equals(id.with)) {
1716 if (isInitiator()) {
1717 if (transition(State.PROCEED)) {
1718 if (serverMsgId != null) {
1719 this.message.setServerMsgId(serverMsgId);
1720 }
1721 this.message.setTime(timestamp);
1722 final Integer remoteDeviceId = proceed.getDeviceId();
1723 if (isOmemoEnabled()) {
1724 this.omemoVerification.setDeviceId(remoteDeviceId);
1725 } else {
1726 if (remoteDeviceId != null) {
1727 Log.d(
1728 Config.LOGTAG,
1729 id.account.getJid().asBareJid()
1730 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1731 }
1732 this.omemoVerification.setDeviceId(null);
1733 }
1734 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1735 } else {
1736 Log.d(
1737 Config.LOGTAG,
1738 String.format(
1739 "%s: ignoring proceed because already in %s",
1740 id.account.getJid().asBareJid(), this.state));
1741 }
1742 } else {
1743 Log.d(
1744 Config.LOGTAG,
1745 String.format(
1746 "%s: ignoring proceed because we were not initializing",
1747 id.account.getJid().asBareJid()));
1748 }
1749 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1750 if (transition(State.ACCEPTED)) {
1751 Log.d(
1752 Config.LOGTAG,
1753 id.account.getJid().asBareJid()
1754 + ": moved session with "
1755 + id.with
1756 + " into state accepted after received carbon copied proceed");
1757 acceptedOnOtherDevice(serverMsgId, timestamp);
1758 }
1759 } else {
1760 Log.d(
1761 Config.LOGTAG,
1762 String.format(
1763 "%s: ignoring proceed from %s. was expected from %s",
1764 id.account.getJid().asBareJid(), from, id.with));
1765 }
1766 }
1767
1768 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1769 if (from.equals(id.with)) {
1770 final State target =
1771 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1772 if (transition(target)) {
1773 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1774 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1775 Log.d(
1776 Config.LOGTAG,
1777 id.account.getJid().asBareJid()
1778 + ": session with "
1779 + id.with
1780 + " has been retracted (serverMsgId="
1781 + serverMsgId
1782 + ")");
1783 if (serverMsgId != null) {
1784 this.message.setServerMsgId(serverMsgId);
1785 }
1786 this.message.setTime(timestamp);
1787 if (target == State.RETRACTED) {
1788 this.message.markUnread();
1789 }
1790 writeLogMessageMissed();
1791 finish();
1792 } else {
1793 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1794 }
1795 } else {
1796 // TODO parse retract from self
1797 Log.d(
1798 Config.LOGTAG,
1799 id.account.getJid().asBareJid()
1800 + ": received retract from "
1801 + from
1802 + ". expected retract from"
1803 + id.with
1804 + ". ignoring");
1805 }
1806 }
1807
1808 public void sendSessionInitiate() {
1809 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1810 }
1811
1812 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1813 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1814 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1815 }
1816
1817 private synchronized void sendSessionInitiate(
1818 final Set<Media> media,
1819 final State targetState,
1820 final List<PeerConnection.IceServer> iceServers) {
1821 if (isTerminated()) {
1822 Log.w(
1823 Config.LOGTAG,
1824 id.account.getJid().asBareJid()
1825 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1826 return;
1827 }
1828 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1829 try {
1830 setupWebRTC(media, iceServers, !includeCandidates);
1831 } catch (final WebRTCWrapper.InitializationException e) {
1832 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1833 webRTCWrapper.close();
1834 sendRetract(Reason.ofThrowable(e));
1835 return;
1836 }
1837 try {
1838 org.webrtc.SessionDescription webRTCSessionDescription =
1839 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1840 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1841 } catch (final Exception e) {
1842 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1843 // exceptions
1844 failureToInitiateSession(e, targetState);
1845 }
1846 }
1847
1848 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1849 if (isTerminated()) {
1850 return;
1851 }
1852 Log.d(
1853 Config.LOGTAG,
1854 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1855 Throwables.getRootCause(throwable));
1856 webRTCWrapper.close();
1857 final Reason reason = Reason.ofThrowable(throwable);
1858 if (isInState(targetState)) {
1859 sendSessionTerminate(reason, throwable.getMessage());
1860 } else {
1861 sendRetract(reason);
1862 }
1863 }
1864
1865 private void sendRetract(final Reason reason) {
1866 // TODO embed reason into retract
1867 sendJingleMessage("retract", id.with.asBareJid());
1868 transitionOrThrow(reasonToState(reason));
1869 this.finish();
1870 }
1871
1872 private void prepareSessionInitiate(
1873 final org.webrtc.SessionDescription webRTCSessionDescription,
1874 final boolean includeCandidates,
1875 final State targetState) {
1876 final SessionDescription sessionDescription =
1877 SessionDescription.parse(webRTCSessionDescription.description);
1878 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1879 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1880 if (includeCandidates) {
1881 candidates = parseCandidates(sessionDescription);
1882 } else {
1883 candidates = ImmutableMultimap.of();
1884 }
1885 this.initiatorRtpContentMap = rtpContentMap;
1886 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1887 encryptSessionInitiate(rtpContentMap);
1888 Futures.addCallback(
1889 outgoingContentMapFuture,
1890 new FutureCallback<>() {
1891 @Override
1892 public void onSuccess(final RtpContentMap outgoingContentMap) {
1893 if (includeCandidates) {
1894 Log.d(
1895 Config.LOGTAG,
1896 "including "
1897 + candidates.size()
1898 + " candidates in session initiate");
1899 sendSessionInitiate(
1900 outgoingContentMap.withCandidates(candidates), targetState);
1901 } else {
1902 sendSessionInitiate(outgoingContentMap, targetState);
1903 }
1904 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1905 }
1906
1907 @Override
1908 public void onFailure(@NonNull final Throwable throwable) {
1909 failureToInitiateSession(throwable, targetState);
1910 }
1911 },
1912 MoreExecutors.directExecutor());
1913 }
1914
1915 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1916 if (isTerminated()) {
1917 Log.w(
1918 Config.LOGTAG,
1919 id.account.getJid().asBareJid()
1920 + ": preparing session was too slow. already terminated. nothing to do.");
1921 return;
1922 }
1923 this.transitionOrThrow(targetState);
1924 final Iq sessionInitiate =
1925 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1926 send(sessionInitiate);
1927 }
1928
1929 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1930 final RtpContentMap rtpContentMap) {
1931 if (this.omemoVerification.hasDeviceId()) {
1932 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1933 verifiedPayloadFuture =
1934 id.account
1935 .getAxolotlService()
1936 .encrypt(
1937 rtpContentMap,
1938 id.with,
1939 omemoVerification.getDeviceId());
1940 final ListenableFuture<RtpContentMap> future =
1941 Futures.transform(
1942 verifiedPayloadFuture,
1943 verifiedPayload -> {
1944 omemoVerification.setSessionFingerprint(
1945 verifiedPayload.getFingerprint());
1946 return verifiedPayload.getPayload();
1947 },
1948 MoreExecutors.directExecutor());
1949 if (Config.REQUIRE_RTP_VERIFICATION) {
1950 return future;
1951 }
1952 return Futures.catching(
1953 future,
1954 CryptoFailedException.class,
1955 e -> {
1956 Log.w(
1957 Config.LOGTAG,
1958 id.account.getJid().asBareJid()
1959 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1960 e);
1961 return rtpContentMap;
1962 },
1963 MoreExecutors.directExecutor());
1964 } else {
1965 return Futures.immediateFuture(rtpContentMap);
1966 }
1967 }
1968
1969 protected void sendSessionTerminate(final Reason reason) {
1970 sendSessionTerminate(reason, null);
1971 }
1972
1973 protected void sendSessionTerminate(final Reason reason, final String text) {
1974 sendSessionTerminate(reason, text, this::writeLogMessage);
1975 sendJingleMessageFinish(reason);
1976 }
1977
1978 private void sendTransportInfo(
1979 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1980 final RtpContentMap transportInfo;
1981 try {
1982 final RtpContentMap rtpContentMap =
1983 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1984 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1985 } catch (final Exception e) {
1986 Log.d(
1987 Config.LOGTAG,
1988 id.account.getJid().asBareJid()
1989 + ": unable to prepare transport-info from candidate for content="
1990 + contentName);
1991 return;
1992 }
1993 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
1994 send(iq);
1995 }
1996
1997 public RtpEndUserState getEndUserState() {
1998 switch (this.state) {
1999 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2000 if (isInitiator()) {
2001 return RtpEndUserState.RINGING;
2002 } else {
2003 return RtpEndUserState.INCOMING_CALL;
2004 }
2005 }
2006 case PROCEED -> {
2007 if (isInitiator()) {
2008 return RtpEndUserState.RINGING;
2009 } else {
2010 return RtpEndUserState.ACCEPTING_CALL;
2011 }
2012 }
2013 case SESSION_INITIALIZED_PRE_APPROVED -> {
2014 if (isInitiator()) {
2015 return RtpEndUserState.RINGING;
2016 } else {
2017 return RtpEndUserState.CONNECTING;
2018 }
2019 }
2020 case SESSION_ACCEPTED -> {
2021 final ContentAddition ca = getPendingContentAddition();
2022 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2023 return RtpEndUserState.INCOMING_CONTENT_ADD;
2024 }
2025 return getPeerConnectionStateAsEndUserState();
2026 }
2027 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2028 if (isInitiator()) {
2029 return RtpEndUserState.DECLINED_OR_BUSY;
2030 } else {
2031 return RtpEndUserState.ENDED;
2032 }
2033 }
2034 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2035 return RtpEndUserState.ENDED;
2036 }
2037 case RETRACTED_RACED -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.ENDED;
2040 } else {
2041 return RtpEndUserState.RETRACTED;
2042 }
2043 }
2044 case TERMINATED_CONNECTIVITY_ERROR -> {
2045 return zeroDuration()
2046 ? RtpEndUserState.CONNECTIVITY_ERROR
2047 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2048 }
2049 case TERMINATED_APPLICATION_FAILURE -> {
2050 return RtpEndUserState.APPLICATION_ERROR;
2051 }
2052 case TERMINATED_SECURITY_ERROR -> {
2053 return RtpEndUserState.SECURITY_ERROR;
2054 }
2055 }
2056 throw new IllegalStateException(
2057 String.format("%s has no equivalent EndUserState", this.state));
2058 }
2059
2060 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2061 final PeerConnection.PeerConnectionState state;
2062 try {
2063 state = webRTCWrapper.getState();
2064 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2065 // We usually close the WebRTCWrapper *before* transitioning so we might still
2066 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2067 return RtpEndUserState.ENDING_CALL;
2068 }
2069 return switch (state) {
2070 case CONNECTED -> RtpEndUserState.CONNECTED;
2071 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2072 case CLOSED -> RtpEndUserState.ENDING_CALL;
2073 default -> zeroDuration()
2074 ? RtpEndUserState.CONNECTIVITY_ERROR
2075 : RtpEndUserState.RECONNECTING;
2076 };
2077 }
2078
2079 private boolean isPeerConnectionConnected() {
2080 try {
2081 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2082 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2083 return false;
2084 }
2085 }
2086
2087 private void updateCallIntegrationState() {
2088 switch (this.state) {
2089 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2090 if (isInitiator()) {
2091 this.callIntegration.setDialing();
2092 } else {
2093 this.callIntegration.setRinging();
2094 }
2095 }
2096 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2097 if (isInitiator()) {
2098 this.callIntegration.setDialing();
2099 } else {
2100 this.callIntegration.setInitialized();
2101 }
2102 }
2103 case SESSION_ACCEPTED -> {
2104 if (isPeerConnectionConnected()) {
2105 this.callIntegration.setActive();
2106 } else {
2107 this.callIntegration.setInitialized();
2108 }
2109 }
2110 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2111 if (isInitiator()) {
2112 this.callIntegration.busy();
2113 } else {
2114 this.callIntegration.rejected();
2115 }
2116 }
2117 case TERMINATED_SUCCESS -> this.callIntegration.success();
2118 case ACCEPTED -> this.callIntegration.accepted();
2119 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2120 .retracted();
2121 case TERMINATED_CONNECTIVITY_ERROR,
2122 TERMINATED_APPLICATION_FAILURE,
2123 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2124 default -> throw new IllegalStateException(
2125 String.format("%s is not handled", this.state));
2126 }
2127 }
2128
2129 public ContentAddition getPendingContentAddition() {
2130 final RtpContentMap in = this.incomingContentAdd;
2131 final RtpContentMap out = this.outgoingContentAdd;
2132 if (out != null) {
2133 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2134 } else if (in != null) {
2135 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2136 } else {
2137 return null;
2138 }
2139 }
2140
2141 public Set<Media> getMedia() {
2142 final State current = getState();
2143 if (current == State.NULL) {
2144 if (isInitiator()) {
2145 return Preconditions.checkNotNull(
2146 this.proposedMedia, "RTP connection has not been initialized properly");
2147 }
2148 throw new IllegalStateException("RTP connection has not been initialized yet");
2149 }
2150 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2151 return Preconditions.checkNotNull(
2152 this.proposedMedia, "RTP connection has not been initialized properly");
2153 }
2154 final RtpContentMap localContentMap = getLocalContentMap();
2155 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2156 if (localContentMap != null) {
2157 return localContentMap.getMedia();
2158 } else if (initiatorContentMap != null) {
2159 return initiatorContentMap.getMedia();
2160 } else if (isTerminated()) {
2161 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2162 } else {
2163 return Preconditions.checkNotNull(
2164 this.proposedMedia, "RTP connection has not been initialized properly");
2165 }
2166 }
2167
2168 public boolean isVerified() {
2169 final String fingerprint = this.omemoVerification.getFingerprint();
2170 if (fingerprint == null) {
2171 return false;
2172 }
2173 final FingerprintStatus status =
2174 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2175 return status != null && status.isVerified();
2176 }
2177
2178 public boolean addMedia(final Media media) {
2179 final Set<Media> currentMedia = getMedia();
2180 if (currentMedia.contains(media)) {
2181 throw new IllegalStateException(String.format("%s has already been proposed", media));
2182 }
2183 // TODO add state protection - can only add while ACCEPTED or so
2184 Log.d(Config.LOGTAG, "adding media: " + media);
2185 return webRTCWrapper.addTrack(media);
2186 }
2187
2188 public synchronized void acceptCall() {
2189 switch (this.state) {
2190 case PROPOSED -> {
2191 cancelRingingTimeout();
2192 acceptCallFromProposed();
2193 }
2194 case SESSION_INITIALIZED -> {
2195 cancelRingingTimeout();
2196 acceptCallFromSessionInitialized();
2197 }
2198 case ACCEPTED -> Log.w(
2199 Config.LOGTAG,
2200 id.account.getJid().asBareJid()
2201 + ": the call has already been accepted with another client. UI was just lagging behind");
2202 case PROCEED, SESSION_ACCEPTED -> Log.w(
2203 Config.LOGTAG,
2204 id.account.getJid().asBareJid()
2205 + ": the call has already been accepted. user probably double tapped the UI");
2206 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2207 }
2208 }
2209
2210 public synchronized void rejectCall() {
2211 if (isTerminated()) {
2212 Log.w(
2213 Config.LOGTAG,
2214 id.account.getJid().asBareJid()
2215 + ": received rejectCall() when session has already been terminated. nothing to do");
2216 return;
2217 }
2218 switch (this.state) {
2219 case PROPOSED -> rejectCallFromProposed();
2220 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2221 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2222 }
2223 }
2224
2225 public synchronized void integrationFailure() {
2226 final var state = getState();
2227 if (state == State.PROPOSED) {
2228 Log.e(
2229 Config.LOGTAG,
2230 id.account.getJid().asBareJid()
2231 + ": failed call integration in state proposed");
2232 rejectCallFromProposed();
2233 } else if (state == State.SESSION_INITIALIZED) {
2234 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2235 this.webRTCWrapper.close();
2236 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2237 } else {
2238 throw new IllegalStateException(
2239 String.format("Can not fail integration in state %s", state));
2240 }
2241 }
2242
2243 public synchronized void endCall() {
2244 if (isTerminated()) {
2245 Log.w(
2246 Config.LOGTAG,
2247 id.account.getJid().asBareJid()
2248 + ": received endCall() when session has already been terminated. nothing to do");
2249 return;
2250 }
2251 if (isInState(State.PROPOSED) && isResponder()) {
2252 rejectCallFromProposed();
2253 return;
2254 }
2255 if (isInState(State.PROCEED)) {
2256 if (isInitiator()) {
2257 retractFromProceed();
2258 } else {
2259 rejectCallFromProceed();
2260 }
2261 return;
2262 }
2263 if (isInitiator()
2264 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2265 this.webRTCWrapper.close();
2266 sendSessionTerminate(Reason.CANCEL);
2267 return;
2268 }
2269 if (isInState(State.SESSION_INITIALIZED)) {
2270 rejectCallFromSessionInitiate();
2271 return;
2272 }
2273 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2274 this.webRTCWrapper.close();
2275 sendSessionTerminate(Reason.SUCCESS);
2276 return;
2277 }
2278 if (isInState(
2279 State.TERMINATED_APPLICATION_FAILURE,
2280 State.TERMINATED_CONNECTIVITY_ERROR,
2281 State.TERMINATED_DECLINED_OR_BUSY)) {
2282 Log.d(
2283 Config.LOGTAG,
2284 "ignoring request to end call because already in state " + this.state);
2285 return;
2286 }
2287 throw new IllegalStateException(
2288 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2289 }
2290
2291 private void retractFromProceed() {
2292 Log.d(Config.LOGTAG, "retract from proceed");
2293 this.sendJingleMessage("retract");
2294 closeTransitionLogFinish(State.RETRACTED_RACED);
2295 }
2296
2297 private void closeTransitionLogFinish(final State state) {
2298 this.webRTCWrapper.close();
2299 transitionOrThrow(state);
2300 writeLogMessage(state);
2301 finish();
2302 }
2303
2304 private void setupWebRTC(
2305 final Set<Media> media,
2306 final List<PeerConnection.IceServer> iceServers,
2307 final boolean trickle)
2308 throws WebRTCWrapper.InitializationException {
2309 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2310 this.webRTCWrapper.setup(this.xmppConnectionService);
2311 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2312 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2313 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2314 }
2315
2316 private void acceptCallFromProposed() {
2317 transitionOrThrow(State.PROCEED);
2318 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2319 this.callIntegration.startAudioRouting();
2320 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2321 this.sendJingleMessage("proceed");
2322 }
2323
2324 private void rejectCallFromProposed() {
2325 transitionOrThrow(State.REJECTED);
2326 writeLogMessageMissed();
2327 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2328 this.sendJingleMessage("reject");
2329 finish();
2330 }
2331
2332 private void rejectCallFromProceed() {
2333 this.sendJingleMessage("reject");
2334 closeTransitionLogFinish(State.REJECTED_RACED);
2335 }
2336
2337 private void rejectCallFromSessionInitiate() {
2338 webRTCWrapper.close();
2339 sendSessionTerminate(Reason.DECLINE);
2340 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2341 }
2342
2343 private void sendJingleMessage(final String action) {
2344 sendJingleMessage(action, id.with);
2345 }
2346
2347 private void sendJingleMessage(final String action, final Jid to) {
2348 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2349 messagePacket.setType(
2350 im.conversations.android.xmpp.model.stanza.Message.Type
2351 .CHAT); // we want to carbon copy those
2352 messagePacket.setTo(to);
2353 final Element intent =
2354 messagePacket
2355 .addChild(action, Namespace.JINGLE_MESSAGE)
2356 .setAttribute("id", id.sessionId);
2357 if ("proceed".equals(action)) {
2358 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2359 if (isOmemoEnabled()) {
2360 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2361 final Element device =
2362 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2363 device.setAttribute("id", deviceId);
2364 }
2365 }
2366 messagePacket.addChild("store", "urn:xmpp:hints");
2367 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2368 }
2369
2370 private void sendJingleMessageFinish(final Reason reason) {
2371 final var account = id.getAccount();
2372 final var messagePacket =
2373 xmppConnectionService
2374 .getMessageGenerator()
2375 .sessionFinish(id.with, id.sessionId, reason);
2376 xmppConnectionService.sendMessagePacket(account, messagePacket);
2377 }
2378
2379 private boolean isOmemoEnabled() {
2380 final Conversational conversational = message.getConversation();
2381 if (conversational instanceof Conversation) {
2382 return ((Conversation) conversational).getNextEncryption()
2383 == Message.ENCRYPTION_AXOLOTL;
2384 }
2385 return false;
2386 }
2387
2388 private void acceptCallFromSessionInitialized() {
2389 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2390 this.callIntegration.startAudioRouting();
2391 sendSessionAccept();
2392 }
2393
2394 @Override
2395 protected synchronized boolean transition(final State target, final Runnable runnable) {
2396 if (super.transition(target, runnable)) {
2397 updateEndUserState();
2398 updateOngoingCallNotification();
2399 return true;
2400 } else {
2401 return false;
2402 }
2403 }
2404
2405 @Override
2406 public void onIceCandidate(final IceCandidate iceCandidate) {
2407 final RtpContentMap rtpContentMap =
2408 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2409 final IceUdpTransportInfo.Credentials credentials;
2410 try {
2411 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2412 } catch (final IllegalArgumentException e) {
2413 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2414 return;
2415 }
2416 final String uFrag = credentials.ufrag;
2417 final IceUdpTransportInfo.Candidate candidate =
2418 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2419 if (candidate == null) {
2420 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2421 return;
2422 }
2423 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2424 sendTransportInfo(iceCandidate.sdpMid, candidate);
2425 }
2426
2427 @Override
2428 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2429 Log.d(
2430 Config.LOGTAG,
2431 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2432 this.stateHistory.add(newState);
2433 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2434 this.sessionDuration.start();
2435 updateOngoingCallNotification();
2436 } else if (this.sessionDuration.isRunning()) {
2437 this.sessionDuration.stop();
2438 updateOngoingCallNotification();
2439 }
2440
2441 final boolean neverConnected =
2442 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2443
2444 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2445 if (neverConnected) {
2446 if (isTerminated()) {
2447 Log.d(
2448 Config.LOGTAG,
2449 id.account.getJid().asBareJid()
2450 + ": not sending session-terminate after connectivity error because session is already in state "
2451 + this.state);
2452 return;
2453 }
2454 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2455 return;
2456 } else {
2457 this.restartIce();
2458 }
2459 }
2460 updateEndUserState();
2461 }
2462
2463 private void restartIce() {
2464 this.stateHistory.clear();
2465 this.webRTCWrapper.restartIceAsync();
2466 }
2467
2468 @Override
2469 public void onRenegotiationNeeded() {
2470 this.webRTCWrapper.execute(this::renegotiate);
2471 }
2472
2473 private synchronized void renegotiate() {
2474 final SessionDescription sessionDescription;
2475 try {
2476 sessionDescription = setLocalSessionDescription();
2477 } catch (final Exception e) {
2478 final Throwable cause = Throwables.getRootCause(e);
2479 webRTCWrapper.close();
2480 if (isTerminated()) {
2481 Log.d(
2482 Config.LOGTAG,
2483 "failed to renegotiate. session was already terminated",
2484 cause);
2485 return;
2486 }
2487 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2488 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2489 return;
2490 }
2491 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2492 final RtpContentMap currentContentMap = getLocalContentMap();
2493 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2494 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2495
2496 Log.d(
2497 Config.LOGTAG,
2498 id.getAccount().getJid().asBareJid()
2499 + ": renegotiate. iceRestart="
2500 + iceRestart
2501 + " content id diff="
2502 + diff);
2503
2504 if (diff.hasModifications() && iceRestart) {
2505 webRTCWrapper.close();
2506 sendSessionTerminate(
2507 Reason.FAILED_APPLICATION,
2508 "WebRTC unexpectedly tried to modify content and transport at once");
2509 return;
2510 }
2511
2512 if (iceRestart) {
2513 initiateIceRestart(rtpContentMap);
2514 return;
2515 } else if (diff.isEmpty()) {
2516 Log.d(
2517 Config.LOGTAG,
2518 "renegotiation. nothing to do. SignalingState="
2519 + this.webRTCWrapper.getSignalingState());
2520 }
2521
2522 if (diff.added.isEmpty()) {
2523 return;
2524 }
2525 modifyLocalContentMap(rtpContentMap);
2526 sendContentAdd(rtpContentMap, diff.added);
2527 }
2528
2529 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2530 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2531 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2532 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2533 iq.setTo(id.with);
2534 xmppConnectionService.sendIqPacket(
2535 id.account,
2536 iq,
2537 (response) -> {
2538 if (response.getType() == Iq.Type.RESULT) {
2539 Log.d(Config.LOGTAG, "received success to our ice restart");
2540 setLocalContentMap(rtpContentMap);
2541 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2542 return;
2543 }
2544 if (response.getType() == Iq.Type.ERROR) {
2545 if (isTieBreak(response)) {
2546 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2547 return;
2548 }
2549 handleIqErrorResponse(response);
2550 }
2551 if (response.getType() == Iq.Type.TIMEOUT) {
2552 handleIqTimeoutResponse(response);
2553 }
2554 });
2555 }
2556
2557 private boolean isTieBreak(final Iq response) {
2558 final Element error = response.findChild("error");
2559 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2560 }
2561
2562 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2563 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2564 this.outgoingContentAdd = contentAdd;
2565 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2566 prepareOutgoingContentMap(contentAdd);
2567 Futures.addCallback(
2568 outgoingContentMapFuture,
2569 new FutureCallback<>() {
2570 @Override
2571 public void onSuccess(final RtpContentMap outgoingContentMap) {
2572 sendContentAdd(outgoingContentMap);
2573 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2574 }
2575
2576 @Override
2577 public void onFailure(@NonNull Throwable throwable) {
2578 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2579 }
2580 },
2581 MoreExecutors.directExecutor());
2582 }
2583
2584 private void sendContentAdd(final RtpContentMap contentAdd) {
2585
2586 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2587 iq.setTo(id.with);
2588 xmppConnectionService.sendIqPacket(
2589 id.account,
2590 iq,
2591 (response) -> {
2592 if (response.getType() == Iq.Type.RESULT) {
2593 Log.d(
2594 Config.LOGTAG,
2595 id.getAccount().getJid().asBareJid()
2596 + ": received ACK to our content-add");
2597 return;
2598 }
2599 if (response.getType() == Iq.Type.ERROR) {
2600 if (isTieBreak(response)) {
2601 this.outgoingContentAdd = null;
2602 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2603 return;
2604 }
2605 handleIqErrorResponse(response);
2606 }
2607 if (response.getType() == Iq.Type.TIMEOUT) {
2608 handleIqTimeoutResponse(response);
2609 }
2610 });
2611 }
2612
2613 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2614 if (isInitiator()) {
2615 this.initiatorRtpContentMap = rtpContentMap;
2616 } else {
2617 this.responderRtpContentMap = rtpContentMap;
2618 }
2619 }
2620
2621 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2622 if (isInitiator()) {
2623 this.responderRtpContentMap = rtpContentMap;
2624 } else {
2625 this.initiatorRtpContentMap = rtpContentMap;
2626 }
2627 }
2628
2629 // this method is to be used for content map modifications that modify media
2630 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2631 final RtpContentMap activeContents = rtpContentMap.activeContents();
2632 setLocalContentMap(activeContents);
2633 this.callIntegration.setAudioDeviceWhenAvailable(
2634 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2635 updateEndUserState();
2636 }
2637
2638 private SessionDescription setLocalSessionDescription()
2639 throws ExecutionException, InterruptedException {
2640 final org.webrtc.SessionDescription sessionDescription =
2641 this.webRTCWrapper.setLocalDescription(false).get();
2642 return SessionDescription.parse(sessionDescription.description);
2643 }
2644
2645 private void closeWebRTCSessionAfterFailedConnection() {
2646 this.webRTCWrapper.close();
2647 synchronized (this) {
2648 if (isTerminated()) {
2649 Log.d(
2650 Config.LOGTAG,
2651 id.account.getJid().asBareJid()
2652 + ": no need to send session-terminate after failed connection. Other party already did");
2653 return;
2654 }
2655 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2656 }
2657 }
2658
2659 public boolean zeroDuration() {
2660 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2661 }
2662
2663 public long getCallDuration() {
2664 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2665 }
2666
2667 @Override
2668 public CallIntegration getCallIntegration() {
2669 return this.callIntegration;
2670 }
2671
2672 public boolean isMicrophoneEnabled() {
2673 return webRTCWrapper.isMicrophoneEnabled();
2674 }
2675
2676 public boolean setMicrophoneEnabled(final boolean enabled) {
2677 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2678 }
2679
2680 public boolean isVideoEnabled() {
2681 return webRTCWrapper.isVideoEnabled();
2682 }
2683
2684 public void setVideoEnabled(final boolean enabled) {
2685 webRTCWrapper.setVideoEnabled(enabled);
2686 }
2687
2688 public boolean isCameraSwitchable() {
2689 return webRTCWrapper.isCameraSwitchable();
2690 }
2691
2692 public boolean isFrontCamera() {
2693 return webRTCWrapper.isFrontCamera();
2694 }
2695
2696 public ListenableFuture<Boolean> switchCamera() {
2697 return webRTCWrapper.switchCamera();
2698 }
2699
2700 @Override
2701 public synchronized void onCallIntegrationShowIncomingCallUi() {
2702 if (isTerminated()) {
2703 // there might be race conditions with the call integration service invoking this
2704 // callback when the rtp session has already ended.
2705 Log.w(
2706 Config.LOGTAG,
2707 "CallIntegration requested incoming call UI but session was already terminated");
2708 return;
2709 }
2710 // TODO apparently this can be called too early as well?
2711 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2712 }
2713
2714 @Override
2715 public void onCallIntegrationDisconnect() {
2716 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2717 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2718 rejectCall();
2719 } else {
2720 endCall();
2721 }
2722 }
2723
2724 @Override
2725 public void onCallIntegrationReject() {
2726 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2727 try {
2728 rejectCall();
2729 } catch (final IllegalStateException e) {
2730 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2731 }
2732 }
2733
2734 @Override
2735 public void onCallIntegrationAnswer() {
2736 // we need to start the UI to a) show it and b) be able to ask for permissions
2737 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2738 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2739 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2740 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2741 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2742 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2743 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2744 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2745 xmppConnectionService.startActivity(intent);
2746 }
2747
2748 @Override
2749 public void onCallIntegrationSilence() {
2750 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2751 }
2752
2753 @Override
2754 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2755 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2756 // that was previous disabled by the user. A proper implementation would probably be to
2757 // track user choice and enable the microphone with a userEnabled() &&
2758 // callIntegration.isMicrophoneEnabled() condition
2759 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2760 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2761 }
2762
2763 @Override
2764 public void onAudioDeviceChanged(
2765 final CallIntegration.AudioDevice selectedAudioDevice,
2766 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2767 Log.d(
2768 Config.LOGTAG,
2769 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2770 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2771 selectedAudioDevice, availableAudioDevices);
2772 }
2773
2774 private void updateEndUserState() {
2775 final RtpEndUserState endUserState = getEndUserState();
2776 this.updateCallIntegrationState();
2777 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2778 id.account, id.with, id.sessionId, endUserState);
2779 }
2780
2781 private void updateOngoingCallNotification() {
2782 final State state = this.state;
2783 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2784 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2785 && isResponder()) {
2786 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2787 xmppConnectionService.removeOngoingCall();
2788 return;
2789 }
2790 final boolean reconnecting;
2791 if (state == State.SESSION_ACCEPTED) {
2792 reconnecting =
2793 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2794 } else {
2795 reconnecting = false;
2796 }
2797 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2798 } else {
2799 xmppConnectionService.removeOngoingCall();
2800 }
2801 }
2802
2803 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2804 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2805 final Iq request = new Iq(Iq.Type.GET);
2806 request.setTo(id.account.getDomain());
2807 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2808 xmppConnectionService.sendIqPacket(
2809 id.account,
2810 request,
2811 (response) -> {
2812 final var iceServers = IceServers.parse(response);
2813 if (iceServers.isEmpty()) {
2814 Log.w(
2815 Config.LOGTAG,
2816 id.account.getJid().asBareJid()
2817 + ": no ICE server found "
2818 + response);
2819 }
2820 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2821 });
2822 } else {
2823 Log.w(
2824 Config.LOGTAG,
2825 id.account.getJid().asBareJid() + ": has no external service discovery");
2826 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2827 }
2828 }
2829
2830 @Override
2831 protected void terminateTransport() {
2832 this.webRTCWrapper.close();
2833 }
2834
2835 @Override
2836 protected void finish() {
2837 if (isTerminated()) {
2838 this.cancelRingingTimeout();
2839 this.callIntegration.verifyDisconnected();
2840 this.webRTCWrapper.verifyClosed();
2841 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2842 super.finish();
2843 } else {
2844 throw new IllegalStateException(
2845 String.format("Unable to call finish from %s", this.state));
2846 }
2847 }
2848
2849 private void writeLogMessage(final State state) {
2850 final long duration = getCallDuration();
2851 if (state == State.TERMINATED_SUCCESS
2852 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2853 writeLogMessageSuccess(duration);
2854 } else {
2855 writeLogMessageMissed();
2856 }
2857 }
2858
2859 private void writeLogMessageSuccess(final long duration) {
2860 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2861 this.writeMessage();
2862 }
2863
2864 private void writeLogMessageMissed() {
2865 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2866 this.writeMessage();
2867 }
2868
2869 private void writeMessage() {
2870 final Conversational conversational = message.getConversation();
2871 if (conversational instanceof Conversation) {
2872 ((Conversation) conversational).add(this.message);
2873 xmppConnectionService.createMessageAsync(message);
2874 xmppConnectionService.updateConversationUi();
2875 } else {
2876 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2877 }
2878 }
2879
2880 public Optional<VideoTrack> getLocalVideoTrack() {
2881 return webRTCWrapper.getLocalVideoTrack();
2882 }
2883
2884 public Optional<VideoTrack> getRemoteVideoTrack() {
2885 return webRTCWrapper.getRemoteVideoTrack();
2886 }
2887
2888 public EglBase.Context getEglBaseContext() {
2889 return webRTCWrapper.getEglBaseContext();
2890 }
2891
2892 void setProposedMedia(final Set<Media> media) {
2893 this.proposedMedia = media;
2894 this.callIntegration.setVideoState(
2895 Media.audioOnly(media)
2896 ? VideoProfile.STATE_AUDIO_ONLY
2897 : VideoProfile.STATE_BIDIRECTIONAL);
2898 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2899 }
2900
2901 public void fireStateUpdate() {
2902 final RtpEndUserState endUserState = getEndUserState();
2903 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2904 id.account, id.with, id.sessionId, endUserState);
2905 }
2906
2907 public boolean isSwitchToVideoAvailable() {
2908 final boolean prerequisite =
2909 Media.audioOnly(getMedia())
2910 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2911 .contains(getEndUserState());
2912 return prerequisite && remoteHasVideoFeature();
2913 }
2914
2915 private boolean remoteHasVideoFeature() {
2916 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2917 }
2918
2919 private boolean remoteHasSdpOfferAnswer() {
2920 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2921 }
2922
2923 @Override
2924 public Account getAccount() {
2925 return id.account;
2926 }
2927
2928 @Override
2929 public Jid getWith() {
2930 return id.with;
2931 }
2932
2933 @Override
2934 public String getSessionId() {
2935 return id.sessionId;
2936 }
2937
2938 private interface OnIceServersDiscovered {
2939 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2940 }
2941}