1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50
51import im.conversations.android.xmpp.model.jingle.Jingle;
52import im.conversations.android.xmpp.model.stanza.Iq;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROPOSED,
78 State.PROCEED,
79 State.SESSION_INITIALIZED_PRE_APPROVED,
80 State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82
83 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
84 private final Queue<
85 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
86 pendingIceCandidates = new LinkedList<>();
87 private final OmemoVerification omemoVerification = new OmemoVerification();
88 private final CallIntegration callIntegration;
89 private final Message message;
90
91 private Set<Media> proposedMedia;
92 private RtpContentMap initiatorRtpContentMap;
93 private RtpContentMap responderRtpContentMap;
94 private RtpContentMap incomingContentAdd;
95 private RtpContentMap outgoingContentAdd;
96 private IceUdpTransportInfo.Setup peerDtlsSetup;
97 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
98 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
99 private ScheduledFuture<?> ringingTimeoutFuture;
100
101 JingleRtpConnection(
102 final JingleConnectionManager jingleConnectionManager,
103 final Id id,
104 final Jid initiator) {
105 this(
106 jingleConnectionManager,
107 id,
108 initiator,
109 new CallIntegration(
110 jingleConnectionManager
111 .getXmppConnectionService()
112 .getApplicationContext()));
113 this.callIntegration.setAddress(
114 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
115 final var contact = id.getContact();
116 this.callIntegration.setCallerDisplayName(
117 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
118 this.callIntegration.setInitialized();
119 }
120
121 JingleRtpConnection(
122 final JingleConnectionManager jingleConnectionManager,
123 final Id id,
124 final Jid initiator,
125 final CallIntegration callIntegration) {
126 super(jingleConnectionManager, id, initiator);
127 final Conversation conversation =
128 jingleConnectionManager
129 .getXmppConnectionService()
130 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
131 this.message =
132 new Message(
133 conversation,
134 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
135 Message.TYPE_RTP_SESSION,
136 id.sessionId);
137 this.callIntegration = callIntegration;
138 this.callIntegration.setCallback(this);
139 }
140
141 @Override
142 synchronized void deliverPacket(final Iq iq) {
143 final var jingle = iq.getExtension(Jingle.class);
144 switch (jingle.getAction()) {
145 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
146 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
147 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
148 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
149 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
150 case CONTENT_ACCEPT -> receiveContentAccept(iq);
151 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
152 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
153 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
154 default -> {
155 respondOk(iq);
156 Log.d(
157 Config.LOGTAG,
158 String.format(
159 "%s: received unhandled jingle action %s",
160 id.account.getJid().asBareJid(), jingle.getAction()));
161 }
162 }
163 }
164
165 @Override
166 synchronized void notifyRebound() {
167 if (isTerminated()) {
168 return;
169 }
170 webRTCWrapper.close();
171 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
172 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
173 }
174 if (isInState(
175 State.SESSION_INITIALIZED,
176 State.SESSION_INITIALIZED_PRE_APPROVED,
177 State.SESSION_ACCEPTED)) {
178 // we might have already changed resources (full jid) at this point; so this might not
179 // even reach the other party
180 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
181 } else {
182 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
183 finish();
184 }
185 }
186
187 private void receiveSessionTerminate(final Iq jinglePacket) {
188 respondOk(jinglePacket);
189 final var jingle = jinglePacket.getExtension(Jingle.class);
190 final Jingle.ReasonWrapper wrapper = jingle.getReason();
191 final State previous = this.state;
192 Log.d(
193 Config.LOGTAG,
194 id.account.getJid().asBareJid()
195 + ": received session terminate reason="
196 + wrapper.reason
197 + "("
198 + Strings.nullToEmpty(wrapper.text)
199 + ") while in state "
200 + previous);
201 if (TERMINATED.contains(previous)) {
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": ignoring session terminate because already in "
206 + previous);
207 return;
208 }
209 webRTCWrapper.close();
210 final State target = reasonToState(wrapper.reason);
211 transitionOrThrow(target);
212 writeLogMessage(target);
213 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
214 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
215 }
216 finish();
217 }
218
219 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
220 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
221 // INITIALIZED only after transport-info has been received
222 if (isInState(
223 State.NULL,
224 State.PROCEED,
225 State.SESSION_INITIALIZED,
226 State.SESSION_INITIALIZED_PRE_APPROVED,
227 State.SESSION_ACCEPTED)) {
228 final RtpContentMap contentMap;
229 try {
230 contentMap = RtpContentMap.of(jingle);
231 } catch (final IllegalArgumentException | NullPointerException e) {
232 Log.d(
233 Config.LOGTAG,
234 id.account.getJid().asBareJid()
235 + ": improperly formatted contents; ignoring",
236 e);
237 respondOk(jinglePacket);
238 return;
239 }
240 receiveTransportInfo(jinglePacket, contentMap);
241 } else {
242 if (isTerminated()) {
243 respondOk(jinglePacket);
244 Log.d(
245 Config.LOGTAG,
246 id.account.getJid().asBareJid()
247 + ": ignoring out-of-order transport info; we where already terminated");
248 } else {
249 Log.d(
250 Config.LOGTAG,
251 id.account.getJid().asBareJid()
252 + ": received transport info while in state="
253 + this.state);
254 terminateWithOutOfOrder(jinglePacket);
255 }
256 }
257 }
258
259 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
260 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
261 candidates = contentMap.contents.entrySet();
262 final RtpContentMap remote = getRemoteContentMap();
263 final Set<String> remoteContentIds =
264 remote == null ? Collections.emptySet() : remote.contents.keySet();
265 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
266 Log.d(
267 Config.LOGTAG,
268 "received transport-info for unknown contents "
269 + contentMap.contents.keySet()
270 + " (known: "
271 + remoteContentIds
272 + ")");
273 respondOk(jinglePacket);
274 pendingIceCandidates.addAll(candidates);
275 return;
276 }
277 if (this.state != State.SESSION_ACCEPTED) {
278 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
279 respondOk(jinglePacket);
280 pendingIceCandidates.addAll(candidates);
281 return;
282 }
283 // zero candidates + modified credentials are an ICE restart offer
284 if (checkForIceRestart(jinglePacket, contentMap)) {
285 return;
286 }
287 respondOk(jinglePacket);
288 try {
289 processCandidates(candidates);
290 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
291 Log.w(
292 Config.LOGTAG,
293 id.account.getJid().asBareJid()
294 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
295 }
296 }
297
298 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
299 final RtpContentMap modification;
300 try {
301 modification = RtpContentMap.of(jingle);
302 modification.requireContentDescriptions();
303 } catch (final RuntimeException e) {
304 Log.d(
305 Config.LOGTAG,
306 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
307 Throwables.getRootCause(e));
308 respondOk(iq);
309 webRTCWrapper.close();
310 sendSessionTerminate(Reason.of(e), e.getMessage());
311 return;
312 }
313 if (isInState(State.SESSION_ACCEPTED)) {
314 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
315 final ListenableFuture<RtpContentMap> future =
316 receiveRtpContentMap(
317 modification,
318 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
319 Futures.addCallback(
320 future,
321 new FutureCallback<>() {
322 @Override
323 public void onSuccess(final RtpContentMap rtpContentMap) {
324 receiveContentAdd(iq, rtpContentMap);
325 }
326
327 @Override
328 public void onFailure(@NonNull Throwable throwable) {
329 respondOk(iq);
330 final Throwable rootCause = Throwables.getRootCause(throwable);
331 Log.d(
332 Config.LOGTAG,
333 id.account.getJid().asBareJid()
334 + ": improperly formatted contents in content-add",
335 throwable);
336 webRTCWrapper.close();
337 sendSessionTerminate(
338 Reason.ofThrowable(rootCause), rootCause.getMessage());
339 }
340 },
341 MoreExecutors.directExecutor());
342 } else {
343 terminateWithOutOfOrder(iq);
344 }
345 }
346
347 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
348 final RtpContentMap remote = getRemoteContentMap();
349 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
350 respondOk(jinglePacket);
351 this.webRTCWrapper.close();
352 sendSessionTerminate(
353 Reason.FAILED_APPLICATION,
354 String.format(
355 "contents with names %s already exists",
356 Joiner.on(", ").join(modification.getNames())));
357 return;
358 }
359 final ContentAddition contentAddition =
360 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
361
362 final RtpContentMap outgoing = this.outgoingContentAdd;
363 final Set<ContentAddition.Summary> outgoingContentAddSummary =
364 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
365
366 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
367 if (isInitiator()) {
368 Log.d(
369 Config.LOGTAG,
370 id.getAccount().getJid().asBareJid()
371 + ": respond with tie break to matching content-add offer");
372 respondWithTieBreak(jinglePacket);
373 } else {
374 Log.d(
375 Config.LOGTAG,
376 id.getAccount().getJid().asBareJid()
377 + ": automatically accept matching content-add offer");
378 acceptContentAdd(contentAddition.summary, modification);
379 }
380 return;
381 }
382
383 // once we can display multiple video tracks we can be more loose with this condition
384 // theoretically it should also be fine to automatically accept audio only contents
385 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
386 Log.d(
387 Config.LOGTAG,
388 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
389 this.incomingContentAdd = modification;
390 respondOk(jinglePacket);
391 updateEndUserState();
392 } else {
393 respondOk(jinglePacket);
394 // TODO do we want to add a reason?
395 rejectContentAdd(modification);
396 }
397 }
398
399 private void receiveContentAccept(final Iq jinglePacket) {
400 final var jingle = jinglePacket.getExtension(Jingle.class);
401 final RtpContentMap receivedContentAccept;
402 try {
403 receivedContentAccept = RtpContentMap.of(jingle);
404 receivedContentAccept.requireContentDescriptions();
405 } catch (final RuntimeException e) {
406 Log.d(
407 Config.LOGTAG,
408 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
409 Throwables.getRootCause(e));
410 respondOk(jinglePacket);
411 webRTCWrapper.close();
412 sendSessionTerminate(Reason.of(e), e.getMessage());
413 return;
414 }
415
416 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
417 if (outgoingContentAdd == null) {
418 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
419 terminateWithOutOfOrder(jinglePacket);
420 return;
421 }
422 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
423 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
424 this.outgoingContentAdd = null;
425 respondOk(jinglePacket);
426 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
427 final ListenableFuture<RtpContentMap> future =
428 receiveRtpContentMap(
429 receivedContentAccept,
430 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
431 Futures.addCallback(
432 future,
433 new FutureCallback<>() {
434 @Override
435 public void onSuccess(final RtpContentMap result) {
436 receiveContentAccept(result);
437 }
438
439 @Override
440 public void onFailure(@NonNull final Throwable throwable) {
441 webRTCWrapper.close();
442 sendSessionTerminate(
443 Reason.ofThrowable(throwable), throwable.getMessage());
444 }
445 },
446 MoreExecutors.directExecutor());
447 } else {
448 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
449 terminateWithOutOfOrder(jinglePacket);
450 }
451 }
452
453 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
454 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
455 final RtpContentMap modifiedContentMap =
456 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
457
458 setRemoteContentMap(modifiedContentMap);
459
460 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
461
462 final org.webrtc.SessionDescription sdp =
463 new org.webrtc.SessionDescription(
464 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
465
466 try {
467 this.webRTCWrapper.setRemoteDescription(sdp).get();
468 } catch (final Exception e) {
469 final Throwable cause = Throwables.getRootCause(e);
470 Log.d(
471 Config.LOGTAG,
472 id.getAccount().getJid().asBareJid()
473 + ": unable to set remote description after receiving content-accept",
474 cause);
475 webRTCWrapper.close();
476 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
477 return;
478 }
479 Log.d(
480 Config.LOGTAG,
481 id.getAccount().getJid().asBareJid()
482 + ": remote has accepted content-add "
483 + ContentAddition.summary(receivedContentAccept));
484 processCandidates(receivedContentAccept.contents.entrySet());
485 updateEndUserState();
486 }
487
488 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
489 if (this.state != State.SESSION_ACCEPTED) {
490 terminateWithOutOfOrder(jinglePacket);
491 return;
492 }
493 final Map<String, Content.Senders> modification =
494 Maps.transformEntries(
495 jingle.getJingleContents(), (key, value) -> value.getSenders());
496 final boolean isInitiator = isInitiator();
497 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
498 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
499 final Set<String> currentOutgoingMediaIds =
500 currentOutgoing == null
501 ? Collections.emptySet()
502 : currentOutgoing.contents.keySet();
503 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
504 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
505 respondOk(jinglePacket);
506 final RtpContentMap modifiedContentMap;
507 try {
508 modifiedContentMap =
509 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
510 } catch (final IllegalArgumentException e) {
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
513 return;
514 }
515 this.outgoingContentAdd = modifiedContentMap;
516 Log.d(
517 Config.LOGTAG,
518 id.account.getJid().asBareJid()
519 + ": processed content-modification for pending content-add");
520 } else if (remoteContentMap != null
521 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
522 respondOk(jinglePacket);
523 final RtpContentMap modifiedRemoteContentMap;
524 try {
525 modifiedRemoteContentMap =
526 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
527 } catch (final IllegalArgumentException e) {
528 webRTCWrapper.close();
529 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
530 return;
531 }
532 final SessionDescription offer;
533 try {
534 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
535 } catch (final IllegalArgumentException | NullPointerException e) {
536 Log.d(
537 Config.LOGTAG,
538 id.getAccount().getJid().asBareJid()
539 + ": unable convert offer from content-modify to SDP",
540 e);
541 webRTCWrapper.close();
542 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
543 return;
544 }
545 Log.d(
546 Config.LOGTAG,
547 id.account.getJid().asBareJid() + ": auto accepting content-modification");
548 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
549 } else {
550 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
551 respondWithItemNotFound(jinglePacket);
552 }
553 }
554
555 private void autoAcceptContentModify(
556 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
557 this.setRemoteContentMap(modifiedRemoteContentMap);
558 final org.webrtc.SessionDescription sdp =
559 new org.webrtc.SessionDescription(
560 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
561 try {
562 this.webRTCWrapper.setRemoteDescription(sdp).get();
563 // auto accept is only done when we already have tracks
564 final SessionDescription answer = setLocalSessionDescription();
565 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
566 modifyLocalContentMap(rtpContentMap);
567 // we do not need to send an answer but do we have to resend the candidates currently in
568 // SDP?
569 // resendCandidatesFromSdp(answer);
570 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
571 } catch (final Exception e) {
572 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
573 webRTCWrapper.close();
574 sendSessionTerminate(Reason.FAILED_APPLICATION);
575 }
576 }
577
578 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
579 final SessionDescription answer) {
580 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
581 new ImmutableMultimap.Builder<>();
582 for (final SessionDescription.Media media : answer.media) {
583 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
584 if (Strings.isNullOrEmpty(mid)) {
585 continue;
586 }
587 for (final String sdpCandidate : media.attributes.get("candidate")) {
588 final IceUdpTransportInfo.Candidate candidate =
589 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
590 if (candidate != null) {
591 candidateBuilder.put(mid, candidate);
592 }
593 }
594 }
595 return candidateBuilder.build();
596 }
597
598 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
599 final RtpContentMap receivedContentReject;
600 try {
601 receivedContentReject = RtpContentMap.of(jingle);
602 } catch (final RuntimeException e) {
603 Log.d(
604 Config.LOGTAG,
605 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
606 Throwables.getRootCause(e));
607 respondOk(jinglePacket);
608 this.webRTCWrapper.close();
609 sendSessionTerminate(Reason.of(e), e.getMessage());
610 return;
611 }
612
613 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
614 if (outgoingContentAdd == null) {
615 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
616 terminateWithOutOfOrder(jinglePacket);
617 return;
618 }
619 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
620 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
621 this.outgoingContentAdd = null;
622 respondOk(jinglePacket);
623 Log.d(Config.LOGTAG, jinglePacket.toString());
624 receiveContentReject(ourSummary);
625 } else {
626 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
627 terminateWithOutOfOrder(jinglePacket);
628 }
629 }
630
631 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
632 try {
633 this.webRTCWrapper.removeTrack(Media.VIDEO);
634 final RtpContentMap localContentMap = customRollback();
635 modifyLocalContentMap(localContentMap);
636 } catch (final Exception e) {
637 final Throwable cause = Throwables.getRootCause(e);
638 Log.d(
639 Config.LOGTAG,
640 id.getAccount().getJid().asBareJid()
641 + ": unable to rollback local description after receiving content-reject",
642 cause);
643 webRTCWrapper.close();
644 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
645 return;
646 }
647 Log.d(
648 Config.LOGTAG,
649 id.getAccount().getJid().asBareJid()
650 + ": remote has rejected our content-add "
651 + summary);
652 }
653
654 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
655 final RtpContentMap receivedContentRemove;
656 try {
657 receivedContentRemove = RtpContentMap.of(jingle);
658 receivedContentRemove.requireContentDescriptions();
659 } catch (final RuntimeException e) {
660 Log.d(
661 Config.LOGTAG,
662 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
663 Throwables.getRootCause(e));
664 respondOk(jinglePacket);
665 this.webRTCWrapper.close();
666 sendSessionTerminate(Reason.of(e), e.getMessage());
667 return;
668 }
669 respondOk(jinglePacket);
670 receiveContentRemove(receivedContentRemove);
671 }
672
673 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
674 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
675 final Set<ContentAddition.Summary> contentAddSummary =
676 incomingContentAdd == null
677 ? Collections.emptySet()
678 : ContentAddition.summary(incomingContentAdd);
679 final Set<ContentAddition.Summary> removeSummary =
680 ContentAddition.summary(receivedContentRemove);
681 if (contentAddSummary.equals(removeSummary)) {
682 this.incomingContentAdd = null;
683 updateEndUserState();
684 } else {
685 webRTCWrapper.close();
686 sendSessionTerminate(
687 Reason.FAILED_APPLICATION,
688 String.format(
689 "%s only supports %s as a means to retract a not yet accepted %s",
690 BuildConfig.APP_NAME,
691 Jingle.Action.CONTENT_REMOVE,
692 Jingle.Action.CONTENT_ADD));
693 }
694 }
695
696 public synchronized void retractContentAdd() {
697 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
698 if (outgoingContentAdd == null) {
699 throw new IllegalStateException("Not outgoing content add");
700 }
701 try {
702 webRTCWrapper.removeTrack(Media.VIDEO);
703 final RtpContentMap localContentMap = customRollback();
704 modifyLocalContentMap(localContentMap);
705 } catch (final Exception e) {
706 final Throwable cause = Throwables.getRootCause(e);
707 Log.d(
708 Config.LOGTAG,
709 id.getAccount().getJid().asBareJid()
710 + ": unable to rollback local description after trying to retract content-add",
711 cause);
712 webRTCWrapper.close();
713 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
714 return;
715 }
716 this.outgoingContentAdd = null;
717 final Iq retract =
718 outgoingContentAdd
719 .toStub()
720 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
721 this.send(retract);
722 Log.d(
723 Config.LOGTAG,
724 id.getAccount().getJid()
725 + ": retract content-add "
726 + ContentAddition.summary(outgoingContentAdd));
727 }
728
729 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
730 final SessionDescription sdp = setLocalSessionDescription();
731 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
732 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
733 this.webRTCWrapper
734 .setRemoteDescription(
735 new org.webrtc.SessionDescription(
736 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
737 .get();
738 return localRtpContentMap;
739 }
740
741 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
742 final RtpContentMap currentRemote = getRemoteContentMap();
743 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
744 if (diff.isEmpty()) {
745 throw new IllegalStateException(
746 "Unexpected rollback condition. No difference between local and remote");
747 }
748 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
749 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
750 final RtpContentMap nextRemote =
751 currentRemote.addContent(
752 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
753 return SessionDescription.of(nextRemote, isResponder());
754 }
755 throw new IllegalStateException(
756 "Unexpected rollback condition. Senders were not uniformly none");
757 }
758
759 public synchronized void acceptContentAdd(
760 @NonNull final Set<ContentAddition.Summary> contentAddition) {
761 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
762 if (incomingContentAdd == null) {
763 throw new IllegalStateException("No incoming content add");
764 }
765
766 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
767 this.incomingContentAdd = null;
768 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
769 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
770 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
771 Log.d(
772 Config.LOGTAG,
773 "content addition is receive only. we want to upgrade to 'both'");
774 final RtpContentMap modifiedSenders =
775 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
776 final Iq proposedContentModification =
777 modifiedSenders
778 .toStub()
779 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
780 proposedContentModification.setTo(id.with);
781 xmppConnectionService.sendIqPacket(
782 id.account,
783 proposedContentModification,
784 (response) -> {
785 if (response.getType() == Iq.Type.RESULT) {
786 Log.d(
787 Config.LOGTAG,
788 id.account.getJid().asBareJid()
789 + ": remote has accepted our upgrade to senders=both");
790 acceptContentAdd(
791 ContentAddition.summary(modifiedSenders), modifiedSenders);
792 } else {
793 Log.d(
794 Config.LOGTAG,
795 id.account.getJid().asBareJid()
796 + ": remote has rejected our upgrade to senders=both");
797 acceptContentAdd(contentAddition, incomingContentAdd);
798 }
799 });
800 } else {
801 acceptContentAdd(contentAddition, incomingContentAdd);
802 }
803 } else {
804 throw new IllegalStateException(
805 "Accepted content add does not match pending content-add");
806 }
807 }
808
809 private void acceptContentAdd(
810 @NonNull final Set<ContentAddition.Summary> contentAddition,
811 final RtpContentMap incomingContentAdd) {
812 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
813 final RtpContentMap modifiedContentMap =
814 getRemoteContentMap().addContent(incomingContentAdd, setup);
815 this.setRemoteContentMap(modifiedContentMap);
816
817 final SessionDescription offer;
818 try {
819 offer = SessionDescription.of(modifiedContentMap, isResponder());
820 } catch (final IllegalArgumentException | NullPointerException e) {
821 Log.d(
822 Config.LOGTAG,
823 id.getAccount().getJid().asBareJid()
824 + ": unable convert offer from content-add to SDP",
825 e);
826 webRTCWrapper.close();
827 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
828 return;
829 }
830 this.incomingContentAdd = null;
831 acceptContentAdd(contentAddition, offer);
832 }
833
834 private void acceptContentAdd(
835 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
836 final org.webrtc.SessionDescription sdp =
837 new org.webrtc.SessionDescription(
838 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
839 try {
840 this.webRTCWrapper.setRemoteDescription(sdp).get();
841
842 // TODO add tracks for 'media' where contentAddition.senders matches
843
844 // TODO if senders.sending(isInitiator())
845
846 this.webRTCWrapper.addTrack(Media.VIDEO);
847
848 // TODO add additional transceivers for recv only cases
849
850 final SessionDescription answer = setLocalSessionDescription();
851 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
852
853 final RtpContentMap contentAcceptMap =
854 rtpContentMap.toContentModification(
855 Collections2.transform(contentAddition, ca -> ca.name));
856
857 Log.d(
858 Config.LOGTAG,
859 id.getAccount().getJid().asBareJid()
860 + ": sending content-accept "
861 + ContentAddition.summary(contentAcceptMap));
862
863 addIceCandidatesFromBlackLog();
864
865 modifyLocalContentMap(rtpContentMap);
866 final ListenableFuture<RtpContentMap> future =
867 prepareOutgoingContentMap(contentAcceptMap);
868 Futures.addCallback(
869 future,
870 new FutureCallback<>() {
871 @Override
872 public void onSuccess(final RtpContentMap rtpContentMap) {
873 sendContentAccept(rtpContentMap);
874 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
875 }
876
877 @Override
878 public void onFailure(@NonNull final Throwable throwable) {
879 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
880 }
881 },
882 MoreExecutors.directExecutor());
883 } catch (final Exception e) {
884 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
885 webRTCWrapper.close();
886 sendSessionTerminate(Reason.FAILED_APPLICATION);
887 }
888 }
889
890 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
891 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
892 send(iq);
893 }
894
895 public synchronized void rejectContentAdd() {
896 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
897 if (incomingContentAdd == null) {
898 throw new IllegalStateException("No incoming content add");
899 }
900 this.incomingContentAdd = null;
901 updateEndUserState();
902 rejectContentAdd(incomingContentAdd);
903 }
904
905 private void rejectContentAdd(final RtpContentMap contentMap) {
906 final Iq iq =
907 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
908 Log.d(
909 Config.LOGTAG,
910 id.getAccount().getJid().asBareJid()
911 + ": rejecting content "
912 + ContentAddition.summary(contentMap));
913 send(iq);
914 }
915
916 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
917 final RtpContentMap existing = getRemoteContentMap();
918 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
919 final IceUdpTransportInfo.Credentials newCredentials;
920 try {
921 existingCredentials = existing.getCredentials();
922 newCredentials = rtpContentMap.getDistinctCredentials();
923 } catch (final IllegalStateException e) {
924 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
925 return false;
926 }
927 if (existingCredentials.contains(newCredentials)) {
928 return false;
929 }
930 // TODO an alternative approach is to check if we already got an iq result to our
931 // ICE-restart
932 // and if that's the case we are seeing an answer.
933 // This might be more spec compliant but also more error prone potentially
934 final boolean isSignalStateStable =
935 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
936 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
937 final boolean isOffer = rtpContentMap.emptyCandidates();
938 final RtpContentMap restartContentMap;
939 try {
940 if (isOffer) {
941 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
942 restartContentMap =
943 existing.modifiedCredentials(
944 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
945 } else {
946 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
947 Log.d(
948 Config.LOGTAG,
949 "received confirmation of ICE restart"
950 + newCredentials
951 + " peer_setup="
952 + setup);
953 // DTLS setup attribute needs to be rewritten to reflect current peer state
954 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
955 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
956 }
957 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
958 return isOffer;
959 } else {
960 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
961 respondWithTieBreak(jinglePacket);
962 return true;
963 }
964 } catch (final Exception exception) {
965 respondOk(jinglePacket);
966 final Throwable rootCause = Throwables.getRootCause(exception);
967 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
968 // If this happens a termination is already in progress
969 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
970 return true;
971 }
972 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
973 webRTCWrapper.close();
974 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
975 return true;
976 }
977 }
978
979 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
980 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
981 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
982 throw new IllegalStateException("Invalid peer setup");
983 }
984 return peerSetup;
985 }
986
987 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
988 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
989 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
990 }
991 this.peerDtlsSetup = setup;
992 }
993
994 private boolean applyIceRestart(
995 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
996 throws ExecutionException, InterruptedException {
997 final SessionDescription sessionDescription =
998 SessionDescription.of(restartContentMap, isResponder());
999 final org.webrtc.SessionDescription.Type type =
1000 isOffer
1001 ? org.webrtc.SessionDescription.Type.OFFER
1002 : org.webrtc.SessionDescription.Type.ANSWER;
1003 org.webrtc.SessionDescription sdp =
1004 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1005 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1006 if (isInitiator()) {
1007 // We ignore the offer and respond with tie-break. This will clause the responder
1008 // not to apply the content map
1009 return false;
1010 }
1011 }
1012 webRTCWrapper.setRemoteDescription(sdp).get();
1013 setRemoteContentMap(restartContentMap);
1014 if (isOffer) {
1015 final SessionDescription localSessionDescription = setLocalSessionDescription();
1016 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1017 // We need to respond OK before sending any candidates
1018 respondOk(jinglePacket);
1019 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1020 } else {
1021 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1022 }
1023 return true;
1024 }
1025
1026 private void processCandidates(
1027 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1028 contents) {
1029 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1030 content : contents) {
1031 processCandidate(content);
1032 }
1033 }
1034
1035 private void processCandidate(
1036 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1037 content) {
1038 final RtpContentMap rtpContentMap = getRemoteContentMap();
1039 final List<String> indices = toIdentificationTags(rtpContentMap);
1040 final String sdpMid = content.getKey(); // aka content name
1041 final IceUdpTransportInfo transport = content.getValue().transport;
1042 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1043
1044 // TODO check that credentials remained the same
1045
1046 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1047 final String sdp;
1048 try {
1049 sdp = candidate.toSdpAttribute(credentials.ufrag);
1050 } catch (final IllegalArgumentException e) {
1051 Log.d(
1052 Config.LOGTAG,
1053 id.account.getJid().asBareJid()
1054 + ": ignoring invalid ICE candidate "
1055 + e.getMessage());
1056 continue;
1057 }
1058 final int mLineIndex = indices.indexOf(sdpMid);
1059 if (mLineIndex < 0) {
1060 Log.w(
1061 Config.LOGTAG,
1062 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1063 }
1064 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1065 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1066 this.webRTCWrapper.addIceCandidate(iceCandidate);
1067 }
1068 }
1069
1070 private RtpContentMap getRemoteContentMap() {
1071 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1072 }
1073
1074 private RtpContentMap getLocalContentMap() {
1075 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1076 }
1077
1078 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1079 final Group originalGroup = rtpContentMap.group;
1080 final List<String> identificationTags =
1081 originalGroup == null
1082 ? rtpContentMap.getNames()
1083 : originalGroup.getIdentificationTags();
1084 if (identificationTags.size() == 0) {
1085 Log.w(
1086 Config.LOGTAG,
1087 id.account.getJid().asBareJid()
1088 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1089 }
1090 return identificationTags;
1091 }
1092
1093 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1094 final Jingle jinglePacket, final boolean expectVerification) {
1095 try {
1096 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1097 } catch (final Exception e) {
1098 return Futures.immediateFailedFuture(e);
1099 }
1100 }
1101
1102 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1103 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1104 Log.d(
1105 Config.LOGTAG,
1106 "receiveRtpContentMap("
1107 + receivedContentMap.getClass().getSimpleName()
1108 + ",expectVerification="
1109 + expectVerification
1110 + ")");
1111 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1112 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1113 id.account
1114 .getAxolotlService()
1115 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1116 return Futures.transform(
1117 future,
1118 omemoVerifiedPayload -> {
1119 // TODO test if an exception here triggers a correct abort
1120 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1121 Log.d(
1122 Config.LOGTAG,
1123 id.account.getJid().asBareJid()
1124 + ": received verifiable DTLS fingerprint via "
1125 + omemoVerification);
1126 return omemoVerifiedPayload.getPayload();
1127 },
1128 MoreExecutors.directExecutor());
1129 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1130 return Futures.immediateFailedFuture(
1131 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1132 } else {
1133 return Futures.immediateFuture(receivedContentMap);
1134 }
1135 }
1136
1137 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1138 if (isInitiator()) {
1139 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1140 return;
1141 }
1142 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1143 Futures.addCallback(
1144 future,
1145 new FutureCallback<>() {
1146 @Override
1147 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1148 receiveSessionInitiate(jinglePacket, rtpContentMap);
1149 }
1150
1151 @Override
1152 public void onFailure(@NonNull final Throwable throwable) {
1153 respondOk(jinglePacket);
1154 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1155 }
1156 },
1157 MoreExecutors.directExecutor());
1158 }
1159
1160 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1161 try {
1162 contentMap.requireContentDescriptions();
1163 contentMap.requireDTLSFingerprint(true);
1164 } catch (final RuntimeException e) {
1165 Log.d(
1166 Config.LOGTAG,
1167 id.account.getJid().asBareJid() + ": improperly formatted contents",
1168 Throwables.getRootCause(e));
1169 respondOk(jinglePacket);
1170 sendSessionTerminate(Reason.of(e), e.getMessage());
1171 return;
1172 }
1173 Log.d(
1174 Config.LOGTAG,
1175 "processing session-init with " + contentMap.contents.size() + " contents");
1176 final State target;
1177 if (this.state == State.PROCEED) {
1178 Preconditions.checkState(
1179 proposedMedia != null && proposedMedia.size() > 0,
1180 "proposed media must be set when processing pre-approved session-initiate");
1181 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1182 sendSessionTerminate(
1183 Reason.SECURITY_ERROR,
1184 String.format(
1185 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1186 this.proposedMedia, contentMap.getMedia()));
1187 return;
1188 }
1189 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1190 } else {
1191 target = State.SESSION_INITIALIZED;
1192 setProposedMedia(contentMap.getMedia());
1193 }
1194 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1195 respondOk(jinglePacket);
1196 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1197 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1198 Log.d(
1199 Config.LOGTAG,
1200 id.account.getJid().asBareJid()
1201 + ": automatically accepting session-initiate");
1202 sendSessionAccept();
1203 } else {
1204 Log.d(
1205 Config.LOGTAG,
1206 id.account.getJid().asBareJid()
1207 + ": received not pre-approved session-initiate. start ringing");
1208 startRinging();
1209 }
1210 } else {
1211 Log.d(
1212 Config.LOGTAG,
1213 String.format(
1214 "%s: received session-initiate while in state %s",
1215 id.account.getJid().asBareJid(), state));
1216 terminateWithOutOfOrder(jinglePacket);
1217 }
1218 }
1219
1220 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1221 if (isResponder()) {
1222 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1223 return;
1224 }
1225 final ListenableFuture<RtpContentMap> future =
1226 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1227 Futures.addCallback(
1228 future,
1229 new FutureCallback<>() {
1230 @Override
1231 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1232 receiveSessionAccept(jinglePacket, rtpContentMap);
1233 }
1234
1235 @Override
1236 public void onFailure(@NonNull final Throwable throwable) {
1237 respondOk(jinglePacket);
1238 Log.d(
1239 Config.LOGTAG,
1240 id.account.getJid().asBareJid()
1241 + ": improperly formatted contents in session-accept",
1242 throwable);
1243 webRTCWrapper.close();
1244 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1245 }
1246 },
1247 MoreExecutors.directExecutor());
1248 }
1249
1250 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1251 try {
1252 contentMap.requireContentDescriptions();
1253 contentMap.requireDTLSFingerprint();
1254 } catch (final RuntimeException e) {
1255 respondOk(jinglePacket);
1256 Log.d(
1257 Config.LOGTAG,
1258 id.account.getJid().asBareJid()
1259 + ": improperly formatted contents in session-accept",
1260 e);
1261 webRTCWrapper.close();
1262 sendSessionTerminate(Reason.of(e), e.getMessage());
1263 return;
1264 }
1265 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1266 if (!initiatorMedia.equals(contentMap.getMedia())) {
1267 sendSessionTerminate(
1268 Reason.SECURITY_ERROR,
1269 String.format(
1270 "Your session-included included media %s but our session-initiate was %s",
1271 this.proposedMedia, contentMap.getMedia()));
1272 return;
1273 }
1274 Log.d(
1275 Config.LOGTAG,
1276 "processing session-accept with " + contentMap.contents.size() + " contents");
1277 if (transition(State.SESSION_ACCEPTED)) {
1278 respondOk(jinglePacket);
1279 receiveSessionAccept(contentMap);
1280 } else {
1281 Log.d(
1282 Config.LOGTAG,
1283 String.format(
1284 "%s: received session-accept while in state %s",
1285 id.account.getJid().asBareJid(), state));
1286 respondOk(jinglePacket);
1287 }
1288 }
1289
1290 private void receiveSessionAccept(final RtpContentMap contentMap) {
1291 this.responderRtpContentMap = contentMap;
1292 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1293 final SessionDescription sessionDescription;
1294 try {
1295 sessionDescription = SessionDescription.of(contentMap, false);
1296 } catch (final IllegalArgumentException | NullPointerException e) {
1297 Log.d(
1298 Config.LOGTAG,
1299 id.account.getJid().asBareJid()
1300 + ": unable convert offer from session-accept to SDP",
1301 e);
1302 webRTCWrapper.close();
1303 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1304 return;
1305 }
1306 final org.webrtc.SessionDescription answer =
1307 new org.webrtc.SessionDescription(
1308 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1309 try {
1310 this.webRTCWrapper.setRemoteDescription(answer).get();
1311 } catch (final Exception e) {
1312 Log.d(
1313 Config.LOGTAG,
1314 id.account.getJid().asBareJid()
1315 + ": unable to set remote description after receiving session-accept",
1316 Throwables.getRootCause(e));
1317 webRTCWrapper.close();
1318 sendSessionTerminate(
1319 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1320 return;
1321 }
1322 processCandidates(contentMap.contents.entrySet());
1323 }
1324
1325 private void sendSessionAccept() {
1326 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1327 if (rtpContentMap == null) {
1328 throw new IllegalStateException("initiator RTP Content Map has not been set");
1329 }
1330 final SessionDescription offer;
1331 try {
1332 offer = SessionDescription.of(rtpContentMap, true);
1333 } catch (final IllegalArgumentException | NullPointerException e) {
1334 Log.d(
1335 Config.LOGTAG,
1336 id.account.getJid().asBareJid()
1337 + ": unable convert offer from session-initiate to SDP",
1338 e);
1339 webRTCWrapper.close();
1340 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1341 return;
1342 }
1343 sendSessionAccept(rtpContentMap.getMedia(), offer);
1344 }
1345
1346 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1347 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1348 }
1349
1350 private synchronized void sendSessionAccept(
1351 final Set<Media> media,
1352 final SessionDescription offer,
1353 final List<PeerConnection.IceServer> iceServers) {
1354 if (isTerminated()) {
1355 Log.w(
1356 Config.LOGTAG,
1357 id.account.getJid().asBareJid()
1358 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1359 return;
1360 }
1361 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1362 try {
1363 setupWebRTC(media, iceServers, !includeCandidates);
1364 } catch (final WebRTCWrapper.InitializationException e) {
1365 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1366 webRTCWrapper.close();
1367 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1368 return;
1369 }
1370 final org.webrtc.SessionDescription sdp =
1371 new org.webrtc.SessionDescription(
1372 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1373 try {
1374 this.webRTCWrapper.setRemoteDescription(sdp).get();
1375 addIceCandidatesFromBlackLog();
1376 org.webrtc.SessionDescription webRTCSessionDescription =
1377 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1378 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1379 } catch (final Exception e) {
1380 failureToAcceptSession(e);
1381 }
1382 }
1383
1384 private void failureToAcceptSession(final Throwable throwable) {
1385 if (isTerminated()) {
1386 return;
1387 }
1388 final Throwable rootCause = Throwables.getRootCause(throwable);
1389 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1390 webRTCWrapper.close();
1391 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1392 }
1393
1394 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1395 if (isTerminated()) {
1396 return;
1397 }
1398 final Throwable rootCause = Throwables.getRootCause(throwable);
1399 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1400 webRTCWrapper.close();
1401 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1402 }
1403
1404 private void addIceCandidatesFromBlackLog() {
1405 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1406 while ((foo = this.pendingIceCandidates.poll()) != null) {
1407 processCandidate(foo);
1408 Log.d(
1409 Config.LOGTAG,
1410 id.account.getJid().asBareJid() + ": added candidate from back log");
1411 }
1412 }
1413
1414 private void prepareSessionAccept(
1415 final org.webrtc.SessionDescription webRTCSessionDescription,
1416 final boolean includeCandidates) {
1417 final SessionDescription sessionDescription =
1418 SessionDescription.parse(webRTCSessionDescription.description);
1419 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1420 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1421 if (includeCandidates) {
1422 candidates = parseCandidates(sessionDescription);
1423 } else {
1424 candidates = ImmutableMultimap.of();
1425 }
1426 this.responderRtpContentMap = respondingRtpContentMap;
1427 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1428 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1429 prepareOutgoingContentMap(respondingRtpContentMap);
1430 Futures.addCallback(
1431 outgoingContentMapFuture,
1432 new FutureCallback<>() {
1433 @Override
1434 public void onSuccess(final RtpContentMap outgoingContentMap) {
1435 if (includeCandidates) {
1436 Log.d(
1437 Config.LOGTAG,
1438 "including "
1439 + candidates.size()
1440 + " candidates in session accept");
1441 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1442 } else {
1443 sendSessionAccept(outgoingContentMap);
1444 }
1445 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1446 }
1447
1448 @Override
1449 public void onFailure(@NonNull Throwable throwable) {
1450 failureToAcceptSession(throwable);
1451 }
1452 },
1453 MoreExecutors.directExecutor());
1454 }
1455
1456 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1457 if (isTerminated()) {
1458 Log.w(
1459 Config.LOGTAG,
1460 id.account.getJid().asBareJid()
1461 + ": preparing session accept was too slow. already terminated. nothing to do.");
1462 return;
1463 }
1464 transitionOrThrow(State.SESSION_ACCEPTED);
1465 final Iq sessionAccept =
1466 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1467 send(sessionAccept);
1468 }
1469
1470 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1471 final RtpContentMap rtpContentMap) {
1472 if (this.omemoVerification.hasDeviceId()) {
1473 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1474 verifiedPayloadFuture =
1475 id.account
1476 .getAxolotlService()
1477 .encrypt(
1478 rtpContentMap,
1479 id.with,
1480 omemoVerification.getDeviceId());
1481 return Futures.transform(
1482 verifiedPayloadFuture,
1483 verifiedPayload -> {
1484 omemoVerification.setOrEnsureEqual(verifiedPayload);
1485 return verifiedPayload.getPayload();
1486 },
1487 MoreExecutors.directExecutor());
1488 } else {
1489 return Futures.immediateFuture(rtpContentMap);
1490 }
1491 }
1492
1493 synchronized void deliveryMessage(
1494 final Jid from,
1495 final Element message,
1496 final String serverMessageId,
1497 final long timestamp) {
1498 Log.d(
1499 Config.LOGTAG,
1500 id.account.getJid().asBareJid()
1501 + ": delivered message to JingleRtpConnection "
1502 + message);
1503 switch (message.getName()) {
1504 case "propose" -> receivePropose(
1505 from, Propose.upgrade(message), serverMessageId, timestamp);
1506 case "proceed" -> receiveProceed(
1507 from, Proceed.upgrade(message), serverMessageId, timestamp);
1508 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1509 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1510 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1511 }
1512 }
1513
1514 void deliverFailedProceed(final String message) {
1515 Log.d(
1516 Config.LOGTAG,
1517 id.account.getJid().asBareJid()
1518 + ": receive message error for proceed message ("
1519 + Strings.nullToEmpty(message)
1520 + ")");
1521 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1522 webRTCWrapper.close();
1523 Log.d(
1524 Config.LOGTAG,
1525 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1526 this.finish();
1527 }
1528 }
1529
1530 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1531 final boolean originatedFromMyself =
1532 from.asBareJid().equals(id.account.getJid().asBareJid());
1533 if (originatedFromMyself) {
1534 if (transition(State.ACCEPTED)) {
1535 acceptedOnOtherDevice(serverMsgId, timestamp);
1536 } else {
1537 Log.d(
1538 Config.LOGTAG,
1539 id.account.getJid().asBareJid()
1540 + ": unable to transition to accept because already in state="
1541 + this.state);
1542 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1543 }
1544 } else {
1545 Log.d(
1546 Config.LOGTAG,
1547 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1548 }
1549 }
1550
1551 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1552 if (serverMsgId != null) {
1553 this.message.setServerMsgId(serverMsgId);
1554 }
1555 this.message.setTime(timestamp);
1556 this.message.setCarbon(true); // indicate that call was accepted on other device
1557 this.writeLogMessageSuccess(0);
1558 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1559 this.finish();
1560 }
1561
1562 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1563 final boolean originatedFromMyself =
1564 from.asBareJid().equals(id.account.getJid().asBareJid());
1565 // reject from another one of my clients
1566 if (originatedFromMyself) {
1567 receiveRejectFromMyself(serverMsgId, timestamp);
1568 } else if (isInitiator()) {
1569 if (from.equals(id.with)) {
1570 receiveRejectFromResponder();
1571 } else {
1572 Log.d(
1573 Config.LOGTAG,
1574 id.account.getJid()
1575 + ": ignoring reject from "
1576 + from
1577 + " for session with "
1578 + id.with);
1579 }
1580 } else {
1581 Log.d(
1582 Config.LOGTAG,
1583 id.account.getJid()
1584 + ": ignoring reject from "
1585 + from
1586 + " for session with "
1587 + id.with);
1588 }
1589 }
1590
1591 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1592 if (transition(State.REJECTED)) {
1593 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1594 this.finish();
1595 if (serverMsgId != null) {
1596 this.message.setServerMsgId(serverMsgId);
1597 }
1598 this.message.setTime(timestamp);
1599 this.message.setCarbon(true); // indicate that call was rejected on other device
1600 writeLogMessageMissed();
1601 } else {
1602 Log.d(
1603 Config.LOGTAG,
1604 "not able to transition into REJECTED because already in " + this.state);
1605 }
1606 }
1607
1608 private void receiveRejectFromResponder() {
1609 if (isInState(State.PROCEED)) {
1610 Log.d(
1611 Config.LOGTAG,
1612 id.account.getJid()
1613 + ": received reject while still in proceed. callee reconsidered");
1614 closeTransitionLogFinish(State.REJECTED_RACED);
1615 return;
1616 }
1617 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1622 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1623 return;
1624 }
1625 Log.d(
1626 Config.LOGTAG,
1627 id.account.getJid()
1628 + ": ignoring reject from responder because already in state "
1629 + this.state);
1630 }
1631
1632 private void receivePropose(
1633 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1634 final boolean originatedFromMyself =
1635 from.asBareJid().equals(id.account.getJid().asBareJid());
1636 if (originatedFromMyself) {
1637 Log.d(
1638 Config.LOGTAG,
1639 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1640 } else if (transition(
1641 State.PROPOSED,
1642 () -> {
1643 final Collection<RtpDescription> descriptions =
1644 Collections2.transform(
1645 Collections2.filter(
1646 propose.getDescriptions(),
1647 d -> d instanceof RtpDescription),
1648 input -> (RtpDescription) input);
1649 final Collection<Media> media =
1650 Collections2.transform(descriptions, RtpDescription::getMedia);
1651 Preconditions.checkState(
1652 !media.contains(Media.UNKNOWN),
1653 "RTP descriptions contain unknown media");
1654 Log.d(
1655 Config.LOGTAG,
1656 id.account.getJid().asBareJid()
1657 + ": received session proposal from "
1658 + from
1659 + " for "
1660 + media);
1661 this.setProposedMedia(Sets.newHashSet(media));
1662 })) {
1663 if (serverMsgId != null) {
1664 this.message.setServerMsgId(serverMsgId);
1665 }
1666 this.message.setTime(timestamp);
1667 startRinging();
1668 // in environments where we always use discovery timeouts we always want to respond with
1669 // 'ringing'
1670 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1671 || (xmppConnectionService.confirmMessages()
1672 && id.getContact().showInContactList())) {
1673 sendJingleMessage("ringing");
1674 }
1675 } else {
1676 Log.d(
1677 Config.LOGTAG,
1678 id.account.getJid()
1679 + ": ignoring session proposal because already in "
1680 + state);
1681 }
1682 }
1683
1684 private void startRinging() {
1685 this.callIntegration.setRinging();
1686 Log.d(
1687 Config.LOGTAG,
1688 id.account.getJid().asBareJid()
1689 + ": received call from "
1690 + id.with
1691 + ". start ringing");
1692 ringingTimeoutFuture =
1693 jingleConnectionManager.schedule(
1694 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1695 if (CallIntegration.selfManaged(xmppConnectionService)) {
1696 return;
1697 }
1698 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1699 }
1700
1701 private synchronized void ringingTimeout() {
1702 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1703 switch (this.state) {
1704 case PROPOSED -> {
1705 message.markUnread();
1706 rejectCallFromProposed();
1707 }
1708 case SESSION_INITIALIZED -> {
1709 message.markUnread();
1710 rejectCallFromSessionInitiate();
1711 }
1712 }
1713 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1714 }
1715
1716 private void cancelRingingTimeout() {
1717 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1718 if (future != null && !future.isCancelled()) {
1719 future.cancel(false);
1720 }
1721 }
1722
1723 private void receiveProceed(
1724 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1725 final Set<Media> media =
1726 Preconditions.checkNotNull(
1727 this.proposedMedia, "Proposed media has to be set before handling proceed");
1728 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1729 if (from.equals(id.with)) {
1730 if (isInitiator()) {
1731 if (transition(State.PROCEED)) {
1732 if (serverMsgId != null) {
1733 this.message.setServerMsgId(serverMsgId);
1734 }
1735 this.message.setTime(timestamp);
1736 final Integer remoteDeviceId = proceed.getDeviceId();
1737 if (isOmemoEnabled()) {
1738 this.omemoVerification.setDeviceId(remoteDeviceId);
1739 } else {
1740 if (remoteDeviceId != null) {
1741 Log.d(
1742 Config.LOGTAG,
1743 id.account.getJid().asBareJid()
1744 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1745 }
1746 this.omemoVerification.setDeviceId(null);
1747 }
1748 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1749 } else {
1750 Log.d(
1751 Config.LOGTAG,
1752 String.format(
1753 "%s: ignoring proceed because already in %s",
1754 id.account.getJid().asBareJid(), this.state));
1755 }
1756 } else {
1757 Log.d(
1758 Config.LOGTAG,
1759 String.format(
1760 "%s: ignoring proceed because we were not initializing",
1761 id.account.getJid().asBareJid()));
1762 }
1763 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1764 if (transition(State.ACCEPTED)) {
1765 Log.d(
1766 Config.LOGTAG,
1767 id.account.getJid().asBareJid()
1768 + ": moved session with "
1769 + id.with
1770 + " into state accepted after received carbon copied proceed");
1771 acceptedOnOtherDevice(serverMsgId, timestamp);
1772 }
1773 } else {
1774 Log.d(
1775 Config.LOGTAG,
1776 String.format(
1777 "%s: ignoring proceed from %s. was expected from %s",
1778 id.account.getJid().asBareJid(), from, id.with));
1779 }
1780 }
1781
1782 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1783 if (from.equals(id.with)) {
1784 final State target =
1785 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1786 if (transition(target)) {
1787 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1788 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1789 Log.d(
1790 Config.LOGTAG,
1791 id.account.getJid().asBareJid()
1792 + ": session with "
1793 + id.with
1794 + " has been retracted (serverMsgId="
1795 + serverMsgId
1796 + ")");
1797 if (serverMsgId != null) {
1798 this.message.setServerMsgId(serverMsgId);
1799 }
1800 this.message.setTime(timestamp);
1801 if (target == State.RETRACTED) {
1802 this.message.markUnread();
1803 }
1804 writeLogMessageMissed();
1805 finish();
1806 } else {
1807 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1808 }
1809 } else {
1810 // TODO parse retract from self
1811 Log.d(
1812 Config.LOGTAG,
1813 id.account.getJid().asBareJid()
1814 + ": received retract from "
1815 + from
1816 + ". expected retract from"
1817 + id.with
1818 + ". ignoring");
1819 }
1820 }
1821
1822 public void sendSessionInitiate() {
1823 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1824 }
1825
1826 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1827 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1828 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1829 }
1830
1831 private synchronized void sendSessionInitiate(
1832 final Set<Media> media,
1833 final State targetState,
1834 final List<PeerConnection.IceServer> iceServers) {
1835 if (isTerminated()) {
1836 Log.w(
1837 Config.LOGTAG,
1838 id.account.getJid().asBareJid()
1839 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1840 return;
1841 }
1842 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1843 try {
1844 setupWebRTC(media, iceServers, !includeCandidates);
1845 } catch (final WebRTCWrapper.InitializationException e) {
1846 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1847 webRTCWrapper.close();
1848 sendRetract(Reason.ofThrowable(e));
1849 return;
1850 }
1851 try {
1852 org.webrtc.SessionDescription webRTCSessionDescription =
1853 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1854 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1855 } catch (final Exception e) {
1856 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1857 // exceptions
1858 failureToInitiateSession(e, targetState);
1859 }
1860 }
1861
1862 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1863 if (isTerminated()) {
1864 return;
1865 }
1866 Log.d(
1867 Config.LOGTAG,
1868 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1869 Throwables.getRootCause(throwable));
1870 webRTCWrapper.close();
1871 final Reason reason = Reason.ofThrowable(throwable);
1872 if (isInState(targetState)) {
1873 sendSessionTerminate(reason, throwable.getMessage());
1874 } else {
1875 sendRetract(reason);
1876 }
1877 }
1878
1879 private void sendRetract(final Reason reason) {
1880 // TODO embed reason into retract
1881 sendJingleMessage("retract", id.with.asBareJid());
1882 transitionOrThrow(reasonToState(reason));
1883 this.finish();
1884 }
1885
1886 private void prepareSessionInitiate(
1887 final org.webrtc.SessionDescription webRTCSessionDescription,
1888 final boolean includeCandidates,
1889 final State targetState) {
1890 final SessionDescription sessionDescription =
1891 SessionDescription.parse(webRTCSessionDescription.description);
1892 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1893 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1894 if (includeCandidates) {
1895 candidates = parseCandidates(sessionDescription);
1896 } else {
1897 candidates = ImmutableMultimap.of();
1898 }
1899 this.initiatorRtpContentMap = rtpContentMap;
1900 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1901 encryptSessionInitiate(rtpContentMap);
1902 Futures.addCallback(
1903 outgoingContentMapFuture,
1904 new FutureCallback<>() {
1905 @Override
1906 public void onSuccess(final RtpContentMap outgoingContentMap) {
1907 if (includeCandidates) {
1908 Log.d(
1909 Config.LOGTAG,
1910 "including "
1911 + candidates.size()
1912 + " candidates in session initiate");
1913 sendSessionInitiate(
1914 outgoingContentMap.withCandidates(candidates), targetState);
1915 } else {
1916 sendSessionInitiate(outgoingContentMap, targetState);
1917 }
1918 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1919 }
1920
1921 @Override
1922 public void onFailure(@NonNull final Throwable throwable) {
1923 failureToInitiateSession(throwable, targetState);
1924 }
1925 },
1926 MoreExecutors.directExecutor());
1927 }
1928
1929 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1930 if (isTerminated()) {
1931 Log.w(
1932 Config.LOGTAG,
1933 id.account.getJid().asBareJid()
1934 + ": preparing session was too slow. already terminated. nothing to do.");
1935 return;
1936 }
1937 this.transitionOrThrow(targetState);
1938 final Iq sessionInitiate =
1939 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1940 send(sessionInitiate);
1941 }
1942
1943 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1944 final RtpContentMap rtpContentMap) {
1945 if (this.omemoVerification.hasDeviceId()) {
1946 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1947 verifiedPayloadFuture =
1948 id.account
1949 .getAxolotlService()
1950 .encrypt(
1951 rtpContentMap,
1952 id.with,
1953 omemoVerification.getDeviceId());
1954 final ListenableFuture<RtpContentMap> future =
1955 Futures.transform(
1956 verifiedPayloadFuture,
1957 verifiedPayload -> {
1958 omemoVerification.setSessionFingerprint(
1959 verifiedPayload.getFingerprint());
1960 return verifiedPayload.getPayload();
1961 },
1962 MoreExecutors.directExecutor());
1963 if (Config.REQUIRE_RTP_VERIFICATION) {
1964 return future;
1965 }
1966 return Futures.catching(
1967 future,
1968 CryptoFailedException.class,
1969 e -> {
1970 Log.w(
1971 Config.LOGTAG,
1972 id.account.getJid().asBareJid()
1973 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1974 e);
1975 return rtpContentMap;
1976 },
1977 MoreExecutors.directExecutor());
1978 } else {
1979 return Futures.immediateFuture(rtpContentMap);
1980 }
1981 }
1982
1983 protected void sendSessionTerminate(final Reason reason) {
1984 sendSessionTerminate(reason, null);
1985 }
1986
1987 protected void sendSessionTerminate(final Reason reason, final String text) {
1988 sendSessionTerminate(reason, text, this::writeLogMessage);
1989 sendJingleMessageFinish(reason);
1990 }
1991
1992 private void sendTransportInfo(
1993 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1994 final RtpContentMap transportInfo;
1995 try {
1996 final RtpContentMap rtpContentMap =
1997 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1998 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1999 } catch (final Exception e) {
2000 Log.d(
2001 Config.LOGTAG,
2002 id.account.getJid().asBareJid()
2003 + ": unable to prepare transport-info from candidate for content="
2004 + contentName);
2005 return;
2006 }
2007 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2008 send(iq);
2009 }
2010
2011 public RtpEndUserState getEndUserState() {
2012 switch (this.state) {
2013 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2014 if (isInitiator()) {
2015 return RtpEndUserState.RINGING;
2016 } else {
2017 return RtpEndUserState.INCOMING_CALL;
2018 }
2019 }
2020 case PROCEED -> {
2021 if (isInitiator()) {
2022 return RtpEndUserState.RINGING;
2023 } else {
2024 return RtpEndUserState.ACCEPTING_CALL;
2025 }
2026 }
2027 case SESSION_INITIALIZED_PRE_APPROVED -> {
2028 if (isInitiator()) {
2029 return RtpEndUserState.RINGING;
2030 } else {
2031 return RtpEndUserState.CONNECTING;
2032 }
2033 }
2034 case SESSION_ACCEPTED -> {
2035 final ContentAddition ca = getPendingContentAddition();
2036 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2037 return RtpEndUserState.INCOMING_CONTENT_ADD;
2038 }
2039 return getPeerConnectionStateAsEndUserState();
2040 }
2041 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2042 if (isInitiator()) {
2043 return RtpEndUserState.DECLINED_OR_BUSY;
2044 } else {
2045 return RtpEndUserState.ENDED;
2046 }
2047 }
2048 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2049 return RtpEndUserState.ENDED;
2050 }
2051 case RETRACTED_RACED -> {
2052 if (isInitiator()) {
2053 return RtpEndUserState.ENDED;
2054 } else {
2055 return RtpEndUserState.RETRACTED;
2056 }
2057 }
2058 case TERMINATED_CONNECTIVITY_ERROR -> {
2059 return zeroDuration()
2060 ? RtpEndUserState.CONNECTIVITY_ERROR
2061 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2062 }
2063 case TERMINATED_APPLICATION_FAILURE -> {
2064 return RtpEndUserState.APPLICATION_ERROR;
2065 }
2066 case TERMINATED_SECURITY_ERROR -> {
2067 return RtpEndUserState.SECURITY_ERROR;
2068 }
2069 }
2070 throw new IllegalStateException(
2071 String.format("%s has no equivalent EndUserState", this.state));
2072 }
2073
2074 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2075 final PeerConnection.PeerConnectionState state;
2076 try {
2077 state = webRTCWrapper.getState();
2078 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2079 // We usually close the WebRTCWrapper *before* transitioning so we might still
2080 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2081 return RtpEndUserState.ENDING_CALL;
2082 }
2083 return switch (state) {
2084 case CONNECTED -> RtpEndUserState.CONNECTED;
2085 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2086 case CLOSED -> RtpEndUserState.ENDING_CALL;
2087 default -> zeroDuration()
2088 ? RtpEndUserState.CONNECTIVITY_ERROR
2089 : RtpEndUserState.RECONNECTING;
2090 };
2091 }
2092
2093 private boolean isPeerConnectionConnected() {
2094 try {
2095 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2096 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2097 return false;
2098 }
2099 }
2100
2101 private void updateCallIntegrationState() {
2102 switch (this.state) {
2103 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2104 if (isInitiator()) {
2105 this.callIntegration.setDialing();
2106 } else {
2107 this.callIntegration.setRinging();
2108 }
2109 }
2110 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2111 if (isInitiator()) {
2112 this.callIntegration.setDialing();
2113 } else {
2114 this.callIntegration.setInitialized();
2115 }
2116 }
2117 case SESSION_ACCEPTED -> {
2118 if (isPeerConnectionConnected()) {
2119 this.callIntegration.setActive();
2120 } else {
2121 this.callIntegration.setInitialized();
2122 }
2123 }
2124 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2125 if (isInitiator()) {
2126 this.callIntegration.busy();
2127 } else {
2128 this.callIntegration.rejected();
2129 }
2130 }
2131 case TERMINATED_SUCCESS -> this.callIntegration.success();
2132 case ACCEPTED -> this.callIntegration.accepted();
2133 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2134 .retracted();
2135 case TERMINATED_CONNECTIVITY_ERROR,
2136 TERMINATED_APPLICATION_FAILURE,
2137 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2138 default -> throw new IllegalStateException(
2139 String.format("%s is not handled", this.state));
2140 }
2141 }
2142
2143 public ContentAddition getPendingContentAddition() {
2144 final RtpContentMap in = this.incomingContentAdd;
2145 final RtpContentMap out = this.outgoingContentAdd;
2146 if (out != null) {
2147 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2148 } else if (in != null) {
2149 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2150 } else {
2151 return null;
2152 }
2153 }
2154
2155 public Set<Media> getMedia() {
2156 final State current = getState();
2157 if (current == State.NULL) {
2158 if (isInitiator()) {
2159 return Preconditions.checkNotNull(
2160 this.proposedMedia, "RTP connection has not been initialized properly");
2161 }
2162 throw new IllegalStateException("RTP connection has not been initialized yet");
2163 }
2164 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2165 return Preconditions.checkNotNull(
2166 this.proposedMedia, "RTP connection has not been initialized properly");
2167 }
2168 final RtpContentMap localContentMap = getLocalContentMap();
2169 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2170 if (localContentMap != null) {
2171 return localContentMap.getMedia();
2172 } else if (initiatorContentMap != null) {
2173 return initiatorContentMap.getMedia();
2174 } else if (isTerminated()) {
2175 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2176 } else {
2177 return Preconditions.checkNotNull(
2178 this.proposedMedia, "RTP connection has not been initialized properly");
2179 }
2180 }
2181
2182 public boolean isVerified() {
2183 final String fingerprint = this.omemoVerification.getFingerprint();
2184 if (fingerprint == null) {
2185 return false;
2186 }
2187 final FingerprintStatus status =
2188 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2189 return status != null && status.isVerified();
2190 }
2191
2192 public boolean addMedia(final Media media) {
2193 final Set<Media> currentMedia = getMedia();
2194 if (currentMedia.contains(media)) {
2195 throw new IllegalStateException(String.format("%s has already been proposed", media));
2196 }
2197 // TODO add state protection - can only add while ACCEPTED or so
2198 Log.d(Config.LOGTAG, "adding media: " + media);
2199 return webRTCWrapper.addTrack(media);
2200 }
2201
2202 public synchronized void acceptCall() {
2203 switch (this.state) {
2204 case PROPOSED -> {
2205 cancelRingingTimeout();
2206 acceptCallFromProposed();
2207 }
2208 case SESSION_INITIALIZED -> {
2209 cancelRingingTimeout();
2210 acceptCallFromSessionInitialized();
2211 }
2212 case ACCEPTED -> Log.w(
2213 Config.LOGTAG,
2214 id.account.getJid().asBareJid()
2215 + ": the call has already been accepted with another client. UI was just lagging behind");
2216 case PROCEED, SESSION_ACCEPTED -> Log.w(
2217 Config.LOGTAG,
2218 id.account.getJid().asBareJid()
2219 + ": the call has already been accepted. user probably double tapped the UI");
2220 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2221 }
2222 }
2223
2224 public synchronized void rejectCall() {
2225 if (isTerminated()) {
2226 Log.w(
2227 Config.LOGTAG,
2228 id.account.getJid().asBareJid()
2229 + ": received rejectCall() when session has already been terminated. nothing to do");
2230 return;
2231 }
2232 switch (this.state) {
2233 case PROPOSED -> rejectCallFromProposed();
2234 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2235 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2236 }
2237 }
2238
2239 public synchronized void integrationFailure() {
2240 final var state = getState();
2241 if (state == State.PROPOSED) {
2242 Log.e(
2243 Config.LOGTAG,
2244 id.account.getJid().asBareJid()
2245 + ": failed call integration in state proposed");
2246 rejectCallFromProposed();
2247 } else if (state == State.SESSION_INITIALIZED) {
2248 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2249 this.webRTCWrapper.close();
2250 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2251 } else {
2252 throw new IllegalStateException(
2253 String.format("Can not fail integration in state %s", state));
2254 }
2255 }
2256
2257 public synchronized void endCall() {
2258 if (isTerminated()) {
2259 Log.w(
2260 Config.LOGTAG,
2261 id.account.getJid().asBareJid()
2262 + ": received endCall() when session has already been terminated. nothing to do");
2263 return;
2264 }
2265 if (isInState(State.PROPOSED) && isResponder()) {
2266 rejectCallFromProposed();
2267 return;
2268 }
2269 if (isInState(State.PROCEED)) {
2270 if (isInitiator()) {
2271 retractFromProceed();
2272 } else {
2273 rejectCallFromProceed();
2274 }
2275 return;
2276 }
2277 if (isInitiator()
2278 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2279 this.webRTCWrapper.close();
2280 sendSessionTerminate(Reason.CANCEL);
2281 return;
2282 }
2283 if (isInState(State.SESSION_INITIALIZED)) {
2284 rejectCallFromSessionInitiate();
2285 return;
2286 }
2287 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2288 this.webRTCWrapper.close();
2289 sendSessionTerminate(Reason.SUCCESS);
2290 return;
2291 }
2292 if (isInState(
2293 State.TERMINATED_APPLICATION_FAILURE,
2294 State.TERMINATED_CONNECTIVITY_ERROR,
2295 State.TERMINATED_DECLINED_OR_BUSY)) {
2296 Log.d(
2297 Config.LOGTAG,
2298 "ignoring request to end call because already in state " + this.state);
2299 return;
2300 }
2301 throw new IllegalStateException(
2302 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2303 }
2304
2305 private void retractFromProceed() {
2306 Log.d(Config.LOGTAG, "retract from proceed");
2307 this.sendJingleMessage("retract");
2308 closeTransitionLogFinish(State.RETRACTED_RACED);
2309 }
2310
2311 private void closeTransitionLogFinish(final State state) {
2312 this.webRTCWrapper.close();
2313 transitionOrThrow(state);
2314 writeLogMessage(state);
2315 finish();
2316 }
2317
2318 private void setupWebRTC(
2319 final Set<Media> media,
2320 final List<PeerConnection.IceServer> iceServers,
2321 final boolean trickle)
2322 throws WebRTCWrapper.InitializationException {
2323 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2324 this.webRTCWrapper.setup(this.xmppConnectionService);
2325 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2326 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2327 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2328 }
2329
2330 private void acceptCallFromProposed() {
2331 transitionOrThrow(State.PROCEED);
2332 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2333 this.callIntegration.startAudioRouting();
2334 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2335 this.sendJingleMessage("proceed");
2336 }
2337
2338 private void rejectCallFromProposed() {
2339 transitionOrThrow(State.REJECTED);
2340 writeLogMessageMissed();
2341 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2342 this.sendJingleMessage("reject");
2343 finish();
2344 }
2345
2346 private void rejectCallFromProceed() {
2347 this.sendJingleMessage("reject");
2348 closeTransitionLogFinish(State.REJECTED_RACED);
2349 }
2350
2351 private void rejectCallFromSessionInitiate() {
2352 webRTCWrapper.close();
2353 sendSessionTerminate(Reason.DECLINE);
2354 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2355 }
2356
2357 private void sendJingleMessage(final String action) {
2358 sendJingleMessage(action, id.with);
2359 }
2360
2361 private void sendJingleMessage(final String action, final Jid to) {
2362 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2363 messagePacket.setType(
2364 im.conversations.android.xmpp.model.stanza.Message.Type
2365 .CHAT); // we want to carbon copy those
2366 messagePacket.setTo(to);
2367 final Element intent =
2368 messagePacket
2369 .addChild(action, Namespace.JINGLE_MESSAGE)
2370 .setAttribute("id", id.sessionId);
2371 if ("proceed".equals(action)) {
2372 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2373 if (isOmemoEnabled()) {
2374 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2375 final Element device =
2376 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2377 device.setAttribute("id", deviceId);
2378 }
2379 }
2380 messagePacket.addChild("store", "urn:xmpp:hints");
2381 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2382 }
2383
2384 private void sendJingleMessageFinish(final Reason reason) {
2385 final var account = id.getAccount();
2386 final var messagePacket =
2387 xmppConnectionService
2388 .getMessageGenerator()
2389 .sessionFinish(id.with, id.sessionId, reason);
2390 xmppConnectionService.sendMessagePacket(account, messagePacket);
2391 }
2392
2393 private boolean isOmemoEnabled() {
2394 final Conversational conversational = message.getConversation();
2395 if (conversational instanceof Conversation) {
2396 return ((Conversation) conversational).getNextEncryption()
2397 == Message.ENCRYPTION_AXOLOTL;
2398 }
2399 return false;
2400 }
2401
2402 private void acceptCallFromSessionInitialized() {
2403 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2404 this.callIntegration.startAudioRouting();
2405 sendSessionAccept();
2406 }
2407
2408 @Override
2409 protected synchronized boolean transition(final State target, final Runnable runnable) {
2410 if (super.transition(target, runnable)) {
2411 updateEndUserState();
2412 updateOngoingCallNotification();
2413 return true;
2414 } else {
2415 return false;
2416 }
2417 }
2418
2419 @Override
2420 public void onIceCandidate(final IceCandidate iceCandidate) {
2421 final RtpContentMap rtpContentMap =
2422 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2423 final IceUdpTransportInfo.Credentials credentials;
2424 try {
2425 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2426 } catch (final IllegalArgumentException e) {
2427 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2428 return;
2429 }
2430 final String uFrag = credentials.ufrag;
2431 final IceUdpTransportInfo.Candidate candidate =
2432 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2433 if (candidate == null) {
2434 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2435 return;
2436 }
2437 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2438 sendTransportInfo(iceCandidate.sdpMid, candidate);
2439 }
2440
2441 @Override
2442 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2443 Log.d(
2444 Config.LOGTAG,
2445 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2446 this.stateHistory.add(newState);
2447 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2448 this.sessionDuration.start();
2449 updateOngoingCallNotification();
2450 } else if (this.sessionDuration.isRunning()) {
2451 this.sessionDuration.stop();
2452 updateOngoingCallNotification();
2453 }
2454
2455 final boolean neverConnected =
2456 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2457
2458 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2459 if (neverConnected) {
2460 if (isTerminated()) {
2461 Log.d(
2462 Config.LOGTAG,
2463 id.account.getJid().asBareJid()
2464 + ": not sending session-terminate after connectivity error because session is already in state "
2465 + this.state);
2466 return;
2467 }
2468 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2469 return;
2470 } else {
2471 this.restartIce();
2472 }
2473 }
2474 updateEndUserState();
2475 }
2476
2477 private void restartIce() {
2478 this.stateHistory.clear();
2479 this.webRTCWrapper.restartIceAsync();
2480 }
2481
2482 @Override
2483 public void onRenegotiationNeeded() {
2484 this.webRTCWrapper.execute(this::renegotiate);
2485 }
2486
2487 private synchronized void renegotiate() {
2488 final SessionDescription sessionDescription;
2489 try {
2490 sessionDescription = setLocalSessionDescription();
2491 } catch (final Exception e) {
2492 final Throwable cause = Throwables.getRootCause(e);
2493 webRTCWrapper.close();
2494 if (isTerminated()) {
2495 Log.d(
2496 Config.LOGTAG,
2497 "failed to renegotiate. session was already terminated",
2498 cause);
2499 return;
2500 }
2501 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2502 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2503 return;
2504 }
2505 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2506 final RtpContentMap currentContentMap = getLocalContentMap();
2507 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2508 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2509
2510 Log.d(
2511 Config.LOGTAG,
2512 id.getAccount().getJid().asBareJid()
2513 + ": renegotiate. iceRestart="
2514 + iceRestart
2515 + " content id diff="
2516 + diff);
2517
2518 if (diff.hasModifications() && iceRestart) {
2519 webRTCWrapper.close();
2520 sendSessionTerminate(
2521 Reason.FAILED_APPLICATION,
2522 "WebRTC unexpectedly tried to modify content and transport at once");
2523 return;
2524 }
2525
2526 if (iceRestart) {
2527 initiateIceRestart(rtpContentMap);
2528 return;
2529 } else if (diff.isEmpty()) {
2530 Log.d(
2531 Config.LOGTAG,
2532 "renegotiation. nothing to do. SignalingState="
2533 + this.webRTCWrapper.getSignalingState());
2534 }
2535
2536 if (diff.added.isEmpty()) {
2537 return;
2538 }
2539 modifyLocalContentMap(rtpContentMap);
2540 sendContentAdd(rtpContentMap, diff.added);
2541 }
2542
2543 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2544 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2545 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2546 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2547 iq.setTo(id.with);
2548 xmppConnectionService.sendIqPacket(
2549 id.account,
2550 iq,
2551 (response) -> {
2552 if (response.getType() == Iq.Type.RESULT) {
2553 Log.d(Config.LOGTAG, "received success to our ice restart");
2554 setLocalContentMap(rtpContentMap);
2555 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2556 return;
2557 }
2558 if (response.getType() == Iq.Type.ERROR) {
2559 if (isTieBreak(response)) {
2560 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2561 return;
2562 }
2563 handleIqErrorResponse(response);
2564 }
2565 if (response.getType() == Iq.Type.TIMEOUT) {
2566 handleIqTimeoutResponse(response);
2567 }
2568 });
2569 }
2570
2571 private boolean isTieBreak(final Iq response) {
2572 final Element error = response.findChild("error");
2573 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2574 }
2575
2576 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2577 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2578 this.outgoingContentAdd = contentAdd;
2579 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2580 prepareOutgoingContentMap(contentAdd);
2581 Futures.addCallback(
2582 outgoingContentMapFuture,
2583 new FutureCallback<>() {
2584 @Override
2585 public void onSuccess(final RtpContentMap outgoingContentMap) {
2586 sendContentAdd(outgoingContentMap);
2587 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2588 }
2589
2590 @Override
2591 public void onFailure(@NonNull Throwable throwable) {
2592 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2593 }
2594 },
2595 MoreExecutors.directExecutor());
2596 }
2597
2598 private void sendContentAdd(final RtpContentMap contentAdd) {
2599
2600 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2601 iq.setTo(id.with);
2602 xmppConnectionService.sendIqPacket(
2603 id.account,
2604 iq,
2605 (response) -> {
2606 if (response.getType() == Iq.Type.RESULT) {
2607 Log.d(
2608 Config.LOGTAG,
2609 id.getAccount().getJid().asBareJid()
2610 + ": received ACK to our content-add");
2611 return;
2612 }
2613 if (response.getType() == Iq.Type.ERROR) {
2614 if (isTieBreak(response)) {
2615 this.outgoingContentAdd = null;
2616 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2617 return;
2618 }
2619 handleIqErrorResponse(response);
2620 }
2621 if (response.getType() == Iq.Type.TIMEOUT) {
2622 handleIqTimeoutResponse(response);
2623 }
2624 });
2625 }
2626
2627 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2628 if (isInitiator()) {
2629 this.initiatorRtpContentMap = rtpContentMap;
2630 } else {
2631 this.responderRtpContentMap = rtpContentMap;
2632 }
2633 }
2634
2635 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2636 if (isInitiator()) {
2637 this.responderRtpContentMap = rtpContentMap;
2638 } else {
2639 this.initiatorRtpContentMap = rtpContentMap;
2640 }
2641 }
2642
2643 // this method is to be used for content map modifications that modify media
2644 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2645 final RtpContentMap activeContents = rtpContentMap.activeContents();
2646 setLocalContentMap(activeContents);
2647 this.callIntegration.setAudioDeviceWhenAvailable(
2648 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2649 updateEndUserState();
2650 }
2651
2652 private SessionDescription setLocalSessionDescription()
2653 throws ExecutionException, InterruptedException {
2654 final org.webrtc.SessionDescription sessionDescription =
2655 this.webRTCWrapper.setLocalDescription(false).get();
2656 return SessionDescription.parse(sessionDescription.description);
2657 }
2658
2659 private void closeWebRTCSessionAfterFailedConnection() {
2660 this.webRTCWrapper.close();
2661 synchronized (this) {
2662 if (isTerminated()) {
2663 Log.d(
2664 Config.LOGTAG,
2665 id.account.getJid().asBareJid()
2666 + ": no need to send session-terminate after failed connection. Other party already did");
2667 return;
2668 }
2669 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2670 }
2671 }
2672
2673 public boolean zeroDuration() {
2674 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2675 }
2676
2677 public long getCallDuration() {
2678 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2679 }
2680
2681 @Override
2682 public CallIntegration getCallIntegration() {
2683 return this.callIntegration;
2684 }
2685
2686 public boolean isMicrophoneEnabled() {
2687 return webRTCWrapper.isMicrophoneEnabled();
2688 }
2689
2690 public boolean setMicrophoneEnabled(final boolean enabled) {
2691 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2692 }
2693
2694 public boolean isVideoEnabled() {
2695 return webRTCWrapper.isVideoEnabled();
2696 }
2697
2698 public void setVideoEnabled(final boolean enabled) {
2699 webRTCWrapper.setVideoEnabled(enabled);
2700 }
2701
2702 public boolean isCameraSwitchable() {
2703 return webRTCWrapper.isCameraSwitchable();
2704 }
2705
2706 public boolean isFrontCamera() {
2707 return webRTCWrapper.isFrontCamera();
2708 }
2709
2710 public ListenableFuture<Boolean> switchCamera() {
2711 return webRTCWrapper.switchCamera();
2712 }
2713
2714 @Override
2715 public synchronized void onCallIntegrationShowIncomingCallUi() {
2716 if (isTerminated()) {
2717 // there might be race conditions with the call integration service invoking this
2718 // callback when the rtp session has already ended.
2719 Log.w(
2720 Config.LOGTAG,
2721 "CallIntegration requested incoming call UI but session was already terminated");
2722 return;
2723 }
2724 // TODO apparently this can be called too early as well?
2725 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2726 }
2727
2728 @Override
2729 public void onCallIntegrationDisconnect() {
2730 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2731 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2732 rejectCall();
2733 } else {
2734 endCall();
2735 }
2736 }
2737
2738 @Override
2739 public void onCallIntegrationReject() {
2740 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2741 try {
2742 rejectCall();
2743 } catch (final IllegalStateException e) {
2744 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2745 }
2746 }
2747
2748 @Override
2749 public void onCallIntegrationAnswer() {
2750 // we need to start the UI to a) show it and b) be able to ask for permissions
2751 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2752 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2753 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2754 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2755 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2756 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2757 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2758 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2759 xmppConnectionService.startActivity(intent);
2760 }
2761
2762 @Override
2763 public void onCallIntegrationSilence() {
2764 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2765 }
2766
2767 @Override
2768 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2769 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2770 // that was previous disabled by the user. A proper implementation would probably be to
2771 // track user choice and enable the microphone with a userEnabled() &&
2772 // callIntegration.isMicrophoneEnabled() condition
2773 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2774 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2775 }
2776
2777 @Override
2778 public void onAudioDeviceChanged(
2779 final CallIntegration.AudioDevice selectedAudioDevice,
2780 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2781 Log.d(
2782 Config.LOGTAG,
2783 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2784 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2785 selectedAudioDevice, availableAudioDevices);
2786 }
2787
2788 private void updateEndUserState() {
2789 final RtpEndUserState endUserState = getEndUserState();
2790 this.updateCallIntegrationState();
2791 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2792 id.account, id.with, id.sessionId, endUserState);
2793 }
2794
2795 private void updateOngoingCallNotification() {
2796 final State state = this.state;
2797 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2798 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2799 && isResponder()) {
2800 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2801 xmppConnectionService.removeOngoingCall();
2802 return;
2803 }
2804 final boolean reconnecting;
2805 if (state == State.SESSION_ACCEPTED) {
2806 reconnecting =
2807 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2808 } else {
2809 reconnecting = false;
2810 }
2811 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2812 } else {
2813 xmppConnectionService.removeOngoingCall();
2814 }
2815 }
2816
2817 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2818 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2819 final Iq request = new Iq(Iq.Type.GET);
2820 request.setTo(id.account.getDomain());
2821 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2822 xmppConnectionService.sendIqPacket(
2823 id.account,
2824 request,
2825 (response) -> {
2826 final var iceServers = IceServers.parse(response);
2827 if (iceServers.isEmpty()) {
2828 Log.w(
2829 Config.LOGTAG,
2830 id.account.getJid().asBareJid()
2831 + ": no ICE server found "
2832 + response);
2833 }
2834 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2835 });
2836 } else {
2837 Log.w(
2838 Config.LOGTAG,
2839 id.account.getJid().asBareJid() + ": has no external service discovery");
2840 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2841 }
2842 }
2843
2844 @Override
2845 protected void terminateTransport() {
2846 this.webRTCWrapper.close();
2847 }
2848
2849 @Override
2850 protected void finish() {
2851 if (isTerminated()) {
2852 this.cancelRingingTimeout();
2853 this.callIntegration.verifyDisconnected();
2854 this.webRTCWrapper.verifyClosed();
2855 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2856 super.finish();
2857 } else {
2858 throw new IllegalStateException(
2859 String.format("Unable to call finish from %s", this.state));
2860 }
2861 }
2862
2863 private void writeLogMessage(final State state) {
2864 final long duration = getCallDuration();
2865 if (state == State.TERMINATED_SUCCESS
2866 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2867 writeLogMessageSuccess(duration);
2868 } else {
2869 writeLogMessageMissed();
2870 }
2871 }
2872
2873 private void writeLogMessageSuccess(final long duration) {
2874 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2875 this.writeMessage();
2876 }
2877
2878 private void writeLogMessageMissed() {
2879 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2880 this.writeMessage();
2881 }
2882
2883 private void writeMessage() {
2884 final Conversational conversational = message.getConversation();
2885 if (conversational instanceof Conversation) {
2886 ((Conversation) conversational).add(this.message);
2887 xmppConnectionService.createMessageAsync(message);
2888 xmppConnectionService.updateConversationUi();
2889 } else {
2890 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2891 }
2892 }
2893
2894 public Optional<VideoTrack> getLocalVideoTrack() {
2895 return webRTCWrapper.getLocalVideoTrack();
2896 }
2897
2898 public Optional<VideoTrack> getRemoteVideoTrack() {
2899 return webRTCWrapper.getRemoteVideoTrack();
2900 }
2901
2902 public EglBase.Context getEglBaseContext() {
2903 return webRTCWrapper.getEglBaseContext();
2904 }
2905
2906 void setProposedMedia(final Set<Media> media) {
2907 this.proposedMedia = media;
2908 this.callIntegration.setVideoState(
2909 Media.audioOnly(media)
2910 ? VideoProfile.STATE_AUDIO_ONLY
2911 : VideoProfile.STATE_BIDIRECTIONAL);
2912 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2913 }
2914
2915 public void fireStateUpdate() {
2916 final RtpEndUserState endUserState = getEndUserState();
2917 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2918 id.account, id.with, id.sessionId, endUserState);
2919 }
2920
2921 public boolean isSwitchToVideoAvailable() {
2922 final boolean prerequisite =
2923 Media.audioOnly(getMedia())
2924 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2925 .contains(getEndUserState());
2926 return prerequisite && remoteHasVideoFeature();
2927 }
2928
2929 private boolean remoteHasVideoFeature() {
2930 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2931 }
2932
2933 private boolean remoteHasSdpOfferAnswer() {
2934 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2935 }
2936
2937 @Override
2938 public Account getAccount() {
2939 return id.account;
2940 }
2941
2942 @Override
2943 public Jid getWith() {
2944 return id.with;
2945 }
2946
2947 @Override
2948 public String getSessionId() {
2949 return id.sessionId;
2950 }
2951
2952 private interface OnIceServersDiscovered {
2953 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2954 }
2955}