1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
50import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
51import eu.siacs.conversations.xmpp.stanzas.IqPacket;
52import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROPOSED,
78 State.PROCEED,
79 State.SESSION_INITIALIZED_PRE_APPROVED,
80 State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82
83 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
84 private final Queue<
85 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
86 pendingIceCandidates = new LinkedList<>();
87 private final OmemoVerification omemoVerification = new OmemoVerification();
88 private final CallIntegration callIntegration;
89 private final Message message;
90
91 private Set<Media> proposedMedia;
92 private RtpContentMap initiatorRtpContentMap;
93 private RtpContentMap responderRtpContentMap;
94 private RtpContentMap incomingContentAdd;
95 private RtpContentMap outgoingContentAdd;
96 private IceUdpTransportInfo.Setup peerDtlsSetup;
97 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
98 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
99 private ScheduledFuture<?> ringingTimeoutFuture;
100
101 JingleRtpConnection(
102 final JingleConnectionManager jingleConnectionManager,
103 final Id id,
104 final Jid initiator) {
105 this(
106 jingleConnectionManager,
107 id,
108 initiator,
109 new CallIntegration(
110 jingleConnectionManager
111 .getXmppConnectionService()
112 .getApplicationContext()));
113 this.callIntegration.setAddress(
114 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
115 final var contact = id.getContact();
116 this.callIntegration.setCallerDisplayName(
117 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
118 this.callIntegration.setInitialized();
119 }
120
121 JingleRtpConnection(
122 final JingleConnectionManager jingleConnectionManager,
123 final Id id,
124 final Jid initiator,
125 final CallIntegration callIntegration) {
126 super(jingleConnectionManager, id, initiator);
127 final Conversation conversation =
128 jingleConnectionManager
129 .getXmppConnectionService()
130 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
131 this.message =
132 new Message(
133 conversation,
134 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
135 Message.TYPE_RTP_SESSION,
136 id.sessionId);
137 this.callIntegration = callIntegration;
138 this.callIntegration.setCallback(this);
139 }
140
141 @Override
142 synchronized void deliverPacket(final JinglePacket jinglePacket) {
143 switch (jinglePacket.getAction()) {
144 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
145 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
146 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
147 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
148 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
149 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
150 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
151 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
152 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
153 default -> {
154 respondOk(jinglePacket);
155 Log.d(
156 Config.LOGTAG,
157 String.format(
158 "%s: received unhandled jingle action %s",
159 id.account.getJid().asBareJid(), jinglePacket.getAction()));
160 }
161 }
162 }
163
164 @Override
165 synchronized void notifyRebound() {
166 if (isTerminated()) {
167 return;
168 }
169 webRTCWrapper.close();
170 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
171 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
172 }
173 if (isInState(
174 State.SESSION_INITIALIZED,
175 State.SESSION_INITIALIZED_PRE_APPROVED,
176 State.SESSION_ACCEPTED)) {
177 // we might have already changed resources (full jid) at this point; so this might not
178 // even reach the other party
179 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
180 } else {
181 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
182 finish();
183 }
184 }
185
186 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
187 respondOk(jinglePacket);
188 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
189 final State previous = this.state;
190 Log.d(
191 Config.LOGTAG,
192 id.account.getJid().asBareJid()
193 + ": received session terminate reason="
194 + wrapper.reason
195 + "("
196 + Strings.nullToEmpty(wrapper.text)
197 + ") while in state "
198 + previous);
199 if (TERMINATED.contains(previous)) {
200 Log.d(
201 Config.LOGTAG,
202 id.account.getJid().asBareJid()
203 + ": ignoring session terminate because already in "
204 + previous);
205 return;
206 }
207 webRTCWrapper.close();
208 final State target = reasonToState(wrapper.reason);
209 transitionOrThrow(target);
210 writeLogMessage(target);
211 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
212 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
213 }
214 finish();
215 }
216
217 private void receiveTransportInfo(final JinglePacket jinglePacket) {
218 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
219 // INITIALIZED only after transport-info has been received
220 if (isInState(
221 State.NULL,
222 State.PROCEED,
223 State.SESSION_INITIALIZED,
224 State.SESSION_INITIALIZED_PRE_APPROVED,
225 State.SESSION_ACCEPTED)) {
226 final RtpContentMap contentMap;
227 try {
228 contentMap = RtpContentMap.of(jinglePacket);
229 } catch (final IllegalArgumentException | NullPointerException e) {
230 Log.d(
231 Config.LOGTAG,
232 id.account.getJid().asBareJid()
233 + ": improperly formatted contents; ignoring",
234 e);
235 respondOk(jinglePacket);
236 return;
237 }
238 receiveTransportInfo(jinglePacket, contentMap);
239 } else {
240 if (isTerminated()) {
241 respondOk(jinglePacket);
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": ignoring out-of-order transport info; we where already terminated");
246 } else {
247 Log.d(
248 Config.LOGTAG,
249 id.account.getJid().asBareJid()
250 + ": received transport info while in state="
251 + this.state);
252 terminateWithOutOfOrder(jinglePacket);
253 }
254 }
255 }
256
257 private void receiveTransportInfo(
258 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
259 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
260 candidates = contentMap.contents.entrySet();
261 final RtpContentMap remote = getRemoteContentMap();
262 final Set<String> remoteContentIds =
263 remote == null ? Collections.emptySet() : remote.contents.keySet();
264 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
265 Log.d(
266 Config.LOGTAG,
267 "received transport-info for unknown contents "
268 + contentMap.contents.keySet()
269 + " (known: "
270 + remoteContentIds
271 + ")");
272 respondOk(jinglePacket);
273 pendingIceCandidates.addAll(candidates);
274 return;
275 }
276 if (this.state != State.SESSION_ACCEPTED) {
277 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
278 respondOk(jinglePacket);
279 pendingIceCandidates.addAll(candidates);
280 return;
281 }
282 // zero candidates + modified credentials are an ICE restart offer
283 if (checkForIceRestart(jinglePacket, contentMap)) {
284 return;
285 }
286 respondOk(jinglePacket);
287 try {
288 processCandidates(candidates);
289 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
290 Log.w(
291 Config.LOGTAG,
292 id.account.getJid().asBareJid()
293 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
294 }
295 }
296
297 private void receiveContentAdd(final JinglePacket jinglePacket) {
298 final RtpContentMap modification;
299 try {
300 modification = RtpContentMap.of(jinglePacket);
301 modification.requireContentDescriptions();
302 } catch (final RuntimeException e) {
303 Log.d(
304 Config.LOGTAG,
305 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
306 Throwables.getRootCause(e));
307 respondOk(jinglePacket);
308 webRTCWrapper.close();
309 sendSessionTerminate(Reason.of(e), e.getMessage());
310 return;
311 }
312 if (isInState(State.SESSION_ACCEPTED)) {
313 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
314 final ListenableFuture<RtpContentMap> future =
315 receiveRtpContentMap(
316 modification,
317 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
318 Futures.addCallback(
319 future,
320 new FutureCallback<>() {
321 @Override
322 public void onSuccess(final RtpContentMap rtpContentMap) {
323 receiveContentAdd(jinglePacket, rtpContentMap);
324 }
325
326 @Override
327 public void onFailure(@NonNull Throwable throwable) {
328 respondOk(jinglePacket);
329 final Throwable rootCause = Throwables.getRootCause(throwable);
330 Log.d(
331 Config.LOGTAG,
332 id.account.getJid().asBareJid()
333 + ": improperly formatted contents in content-add",
334 throwable);
335 webRTCWrapper.close();
336 sendSessionTerminate(
337 Reason.ofThrowable(rootCause), rootCause.getMessage());
338 }
339 },
340 MoreExecutors.directExecutor());
341 } else {
342 terminateWithOutOfOrder(jinglePacket);
343 }
344 }
345
346 private void receiveContentAdd(
347 final JinglePacket jinglePacket, final RtpContentMap modification) {
348 final RtpContentMap remote = getRemoteContentMap();
349 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
350 respondOk(jinglePacket);
351 this.webRTCWrapper.close();
352 sendSessionTerminate(
353 Reason.FAILED_APPLICATION,
354 String.format(
355 "contents with names %s already exists",
356 Joiner.on(", ").join(modification.getNames())));
357 return;
358 }
359 final ContentAddition contentAddition =
360 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
361
362 final RtpContentMap outgoing = this.outgoingContentAdd;
363 final Set<ContentAddition.Summary> outgoingContentAddSummary =
364 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
365
366 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
367 if (isInitiator()) {
368 Log.d(
369 Config.LOGTAG,
370 id.getAccount().getJid().asBareJid()
371 + ": respond with tie break to matching content-add offer");
372 respondWithTieBreak(jinglePacket);
373 } else {
374 Log.d(
375 Config.LOGTAG,
376 id.getAccount().getJid().asBareJid()
377 + ": automatically accept matching content-add offer");
378 acceptContentAdd(contentAddition.summary, modification);
379 }
380 return;
381 }
382
383 // once we can display multiple video tracks we can be more loose with this condition
384 // theoretically it should also be fine to automatically accept audio only contents
385 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
386 Log.d(
387 Config.LOGTAG,
388 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
389 this.incomingContentAdd = modification;
390 respondOk(jinglePacket);
391 updateEndUserState();
392 } else {
393 respondOk(jinglePacket);
394 // TODO do we want to add a reason?
395 rejectContentAdd(modification);
396 }
397 }
398
399 private void receiveContentAccept(final JinglePacket jinglePacket) {
400 final RtpContentMap receivedContentAccept;
401 try {
402 receivedContentAccept = RtpContentMap.of(jinglePacket);
403 receivedContentAccept.requireContentDescriptions();
404 } catch (final RuntimeException e) {
405 Log.d(
406 Config.LOGTAG,
407 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
408 Throwables.getRootCause(e));
409 respondOk(jinglePacket);
410 webRTCWrapper.close();
411 sendSessionTerminate(Reason.of(e), e.getMessage());
412 return;
413 }
414
415 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
416 if (outgoingContentAdd == null) {
417 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
418 terminateWithOutOfOrder(jinglePacket);
419 return;
420 }
421 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
422 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
423 this.outgoingContentAdd = null;
424 respondOk(jinglePacket);
425 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
426 final ListenableFuture<RtpContentMap> future =
427 receiveRtpContentMap(
428 receivedContentAccept,
429 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
430 Futures.addCallback(
431 future,
432 new FutureCallback<>() {
433 @Override
434 public void onSuccess(final RtpContentMap result) {
435 receiveContentAccept(result);
436 }
437
438 @Override
439 public void onFailure(@NonNull final Throwable throwable) {
440 webRTCWrapper.close();
441 sendSessionTerminate(
442 Reason.ofThrowable(throwable), throwable.getMessage());
443 }
444 },
445 MoreExecutors.directExecutor());
446 } else {
447 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
448 terminateWithOutOfOrder(jinglePacket);
449 }
450 }
451
452 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
453 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
454 final RtpContentMap modifiedContentMap =
455 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
456
457 setRemoteContentMap(modifiedContentMap);
458
459 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
460
461 final org.webrtc.SessionDescription sdp =
462 new org.webrtc.SessionDescription(
463 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
464
465 try {
466 this.webRTCWrapper.setRemoteDescription(sdp).get();
467 } catch (final Exception e) {
468 final Throwable cause = Throwables.getRootCause(e);
469 Log.d(
470 Config.LOGTAG,
471 id.getAccount().getJid().asBareJid()
472 + ": unable to set remote description after receiving content-accept",
473 cause);
474 webRTCWrapper.close();
475 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
476 return;
477 }
478 Log.d(
479 Config.LOGTAG,
480 id.getAccount().getJid().asBareJid()
481 + ": remote has accepted content-add "
482 + ContentAddition.summary(receivedContentAccept));
483 processCandidates(receivedContentAccept.contents.entrySet());
484 updateEndUserState();
485 }
486
487 private void receiveContentModify(final JinglePacket jinglePacket) {
488 if (this.state != State.SESSION_ACCEPTED) {
489 terminateWithOutOfOrder(jinglePacket);
490 return;
491 }
492 final Map<String, Content.Senders> modification =
493 Maps.transformEntries(
494 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
495 final boolean isInitiator = isInitiator();
496 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
497 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
498 final Set<String> currentOutgoingMediaIds =
499 currentOutgoing == null
500 ? Collections.emptySet()
501 : currentOutgoing.contents.keySet();
502 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
503 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
504 respondOk(jinglePacket);
505 final RtpContentMap modifiedContentMap;
506 try {
507 modifiedContentMap =
508 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
509 } catch (final IllegalArgumentException e) {
510 webRTCWrapper.close();
511 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
512 return;
513 }
514 this.outgoingContentAdd = modifiedContentMap;
515 Log.d(
516 Config.LOGTAG,
517 id.account.getJid().asBareJid()
518 + ": processed content-modification for pending content-add");
519 } else if (remoteContentMap != null
520 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
521 respondOk(jinglePacket);
522 final RtpContentMap modifiedRemoteContentMap;
523 try {
524 modifiedRemoteContentMap =
525 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
526 } catch (final IllegalArgumentException e) {
527 webRTCWrapper.close();
528 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
529 return;
530 }
531 final SessionDescription offer;
532 try {
533 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
534 } catch (final IllegalArgumentException | NullPointerException e) {
535 Log.d(
536 Config.LOGTAG,
537 id.getAccount().getJid().asBareJid()
538 + ": unable convert offer from content-modify to SDP",
539 e);
540 webRTCWrapper.close();
541 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
542 return;
543 }
544 Log.d(
545 Config.LOGTAG,
546 id.account.getJid().asBareJid() + ": auto accepting content-modification");
547 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
548 } else {
549 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
550 respondWithItemNotFound(jinglePacket);
551 }
552 }
553
554 private void autoAcceptContentModify(
555 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
556 this.setRemoteContentMap(modifiedRemoteContentMap);
557 final org.webrtc.SessionDescription sdp =
558 new org.webrtc.SessionDescription(
559 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
560 try {
561 this.webRTCWrapper.setRemoteDescription(sdp).get();
562 // auto accept is only done when we already have tracks
563 final SessionDescription answer = setLocalSessionDescription();
564 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
565 modifyLocalContentMap(rtpContentMap);
566 // we do not need to send an answer but do we have to resend the candidates currently in
567 // SDP?
568 // resendCandidatesFromSdp(answer);
569 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
570 } catch (final Exception e) {
571 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
572 webRTCWrapper.close();
573 sendSessionTerminate(Reason.FAILED_APPLICATION);
574 }
575 }
576
577 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
578 final SessionDescription answer) {
579 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
580 new ImmutableMultimap.Builder<>();
581 for (final SessionDescription.Media media : answer.media) {
582 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
583 if (Strings.isNullOrEmpty(mid)) {
584 continue;
585 }
586 for (final String sdpCandidate : media.attributes.get("candidate")) {
587 final IceUdpTransportInfo.Candidate candidate =
588 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
589 if (candidate != null) {
590 candidateBuilder.put(mid, candidate);
591 }
592 }
593 }
594 return candidateBuilder.build();
595 }
596
597 private void receiveContentReject(final JinglePacket jinglePacket) {
598 final RtpContentMap receivedContentReject;
599 try {
600 receivedContentReject = RtpContentMap.of(jinglePacket);
601 } catch (final RuntimeException e) {
602 Log.d(
603 Config.LOGTAG,
604 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
605 Throwables.getRootCause(e));
606 respondOk(jinglePacket);
607 this.webRTCWrapper.close();
608 sendSessionTerminate(Reason.of(e), e.getMessage());
609 return;
610 }
611
612 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
613 if (outgoingContentAdd == null) {
614 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
615 terminateWithOutOfOrder(jinglePacket);
616 return;
617 }
618 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
619 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
620 this.outgoingContentAdd = null;
621 respondOk(jinglePacket);
622 Log.d(Config.LOGTAG, jinglePacket.toString());
623 receiveContentReject(ourSummary);
624 } else {
625 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
626 terminateWithOutOfOrder(jinglePacket);
627 }
628 }
629
630 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
631 try {
632 this.webRTCWrapper.removeTrack(Media.VIDEO);
633 final RtpContentMap localContentMap = customRollback();
634 modifyLocalContentMap(localContentMap);
635 } catch (final Exception e) {
636 final Throwable cause = Throwables.getRootCause(e);
637 Log.d(
638 Config.LOGTAG,
639 id.getAccount().getJid().asBareJid()
640 + ": unable to rollback local description after receiving content-reject",
641 cause);
642 webRTCWrapper.close();
643 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
644 return;
645 }
646 Log.d(
647 Config.LOGTAG,
648 id.getAccount().getJid().asBareJid()
649 + ": remote has rejected our content-add "
650 + summary);
651 }
652
653 private void receiveContentRemove(final JinglePacket jinglePacket) {
654 final RtpContentMap receivedContentRemove;
655 try {
656 receivedContentRemove = RtpContentMap.of(jinglePacket);
657 receivedContentRemove.requireContentDescriptions();
658 } catch (final RuntimeException e) {
659 Log.d(
660 Config.LOGTAG,
661 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
662 Throwables.getRootCause(e));
663 respondOk(jinglePacket);
664 this.webRTCWrapper.close();
665 sendSessionTerminate(Reason.of(e), e.getMessage());
666 return;
667 }
668 respondOk(jinglePacket);
669 receiveContentRemove(receivedContentRemove);
670 }
671
672 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
673 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
674 final Set<ContentAddition.Summary> contentAddSummary =
675 incomingContentAdd == null
676 ? Collections.emptySet()
677 : ContentAddition.summary(incomingContentAdd);
678 final Set<ContentAddition.Summary> removeSummary =
679 ContentAddition.summary(receivedContentRemove);
680 if (contentAddSummary.equals(removeSummary)) {
681 this.incomingContentAdd = null;
682 updateEndUserState();
683 } else {
684 webRTCWrapper.close();
685 sendSessionTerminate(
686 Reason.FAILED_APPLICATION,
687 String.format(
688 "%s only supports %s as a means to retract a not yet accepted %s",
689 BuildConfig.APP_NAME,
690 JinglePacket.Action.CONTENT_REMOVE,
691 JinglePacket.Action.CONTENT_ADD));
692 }
693 }
694
695 public synchronized void retractContentAdd() {
696 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
697 if (outgoingContentAdd == null) {
698 throw new IllegalStateException("Not outgoing content add");
699 }
700 try {
701 webRTCWrapper.removeTrack(Media.VIDEO);
702 final RtpContentMap localContentMap = customRollback();
703 modifyLocalContentMap(localContentMap);
704 } catch (final Exception e) {
705 final Throwable cause = Throwables.getRootCause(e);
706 Log.d(
707 Config.LOGTAG,
708 id.getAccount().getJid().asBareJid()
709 + ": unable to rollback local description after trying to retract content-add",
710 cause);
711 webRTCWrapper.close();
712 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
713 return;
714 }
715 this.outgoingContentAdd = null;
716 final JinglePacket retract =
717 outgoingContentAdd
718 .toStub()
719 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
720 this.send(retract);
721 Log.d(
722 Config.LOGTAG,
723 id.getAccount().getJid()
724 + ": retract content-add "
725 + ContentAddition.summary(outgoingContentAdd));
726 }
727
728 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
729 final SessionDescription sdp = setLocalSessionDescription();
730 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
731 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
732 this.webRTCWrapper
733 .setRemoteDescription(
734 new org.webrtc.SessionDescription(
735 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
736 .get();
737 return localRtpContentMap;
738 }
739
740 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
741 final RtpContentMap currentRemote = getRemoteContentMap();
742 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
743 if (diff.isEmpty()) {
744 throw new IllegalStateException(
745 "Unexpected rollback condition. No difference between local and remote");
746 }
747 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
748 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
749 final RtpContentMap nextRemote =
750 currentRemote.addContent(
751 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
752 return SessionDescription.of(nextRemote, isResponder());
753 }
754 throw new IllegalStateException(
755 "Unexpected rollback condition. Senders were not uniformly none");
756 }
757
758 public synchronized void acceptContentAdd(
759 @NonNull final Set<ContentAddition.Summary> contentAddition) {
760 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
761 if (incomingContentAdd == null) {
762 throw new IllegalStateException("No incoming content add");
763 }
764
765 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
766 this.incomingContentAdd = null;
767 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
768 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
769 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
770 Log.d(
771 Config.LOGTAG,
772 "content addition is receive only. we want to upgrade to 'both'");
773 final RtpContentMap modifiedSenders =
774 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
775 final JinglePacket proposedContentModification =
776 modifiedSenders
777 .toStub()
778 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
779 proposedContentModification.setTo(id.with);
780 xmppConnectionService.sendIqPacket(
781 id.account,
782 proposedContentModification,
783 (account, response) -> {
784 if (response.getType() == IqPacket.TYPE.RESULT) {
785 Log.d(
786 Config.LOGTAG,
787 id.account.getJid().asBareJid()
788 + ": remote has accepted our upgrade to senders=both");
789 acceptContentAdd(
790 ContentAddition.summary(modifiedSenders), modifiedSenders);
791 } else {
792 Log.d(
793 Config.LOGTAG,
794 id.account.getJid().asBareJid()
795 + ": remote has rejected our upgrade to senders=both");
796 acceptContentAdd(contentAddition, incomingContentAdd);
797 }
798 });
799 } else {
800 acceptContentAdd(contentAddition, incomingContentAdd);
801 }
802 } else {
803 throw new IllegalStateException(
804 "Accepted content add does not match pending content-add");
805 }
806 }
807
808 private void acceptContentAdd(
809 @NonNull final Set<ContentAddition.Summary> contentAddition,
810 final RtpContentMap incomingContentAdd) {
811 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
812 final RtpContentMap modifiedContentMap =
813 getRemoteContentMap().addContent(incomingContentAdd, setup);
814 this.setRemoteContentMap(modifiedContentMap);
815
816 final SessionDescription offer;
817 try {
818 offer = SessionDescription.of(modifiedContentMap, isResponder());
819 } catch (final IllegalArgumentException | NullPointerException e) {
820 Log.d(
821 Config.LOGTAG,
822 id.getAccount().getJid().asBareJid()
823 + ": unable convert offer from content-add to SDP",
824 e);
825 webRTCWrapper.close();
826 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
827 return;
828 }
829 this.incomingContentAdd = null;
830 acceptContentAdd(contentAddition, offer);
831 }
832
833 private void acceptContentAdd(
834 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
835 final org.webrtc.SessionDescription sdp =
836 new org.webrtc.SessionDescription(
837 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
838 try {
839 this.webRTCWrapper.setRemoteDescription(sdp).get();
840
841 // TODO add tracks for 'media' where contentAddition.senders matches
842
843 // TODO if senders.sending(isInitiator())
844
845 this.webRTCWrapper.addTrack(Media.VIDEO);
846
847 // TODO add additional transceivers for recv only cases
848
849 final SessionDescription answer = setLocalSessionDescription();
850 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
851
852 final RtpContentMap contentAcceptMap =
853 rtpContentMap.toContentModification(
854 Collections2.transform(contentAddition, ca -> ca.name));
855
856 Log.d(
857 Config.LOGTAG,
858 id.getAccount().getJid().asBareJid()
859 + ": sending content-accept "
860 + ContentAddition.summary(contentAcceptMap));
861
862 addIceCandidatesFromBlackLog();
863
864 modifyLocalContentMap(rtpContentMap);
865 final ListenableFuture<RtpContentMap> future =
866 prepareOutgoingContentMap(contentAcceptMap);
867 Futures.addCallback(
868 future,
869 new FutureCallback<>() {
870 @Override
871 public void onSuccess(final RtpContentMap rtpContentMap) {
872 sendContentAccept(rtpContentMap);
873 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
874 }
875
876 @Override
877 public void onFailure(@NonNull final Throwable throwable) {
878 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
879 }
880 },
881 MoreExecutors.directExecutor());
882 } catch (final Exception e) {
883 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
884 webRTCWrapper.close();
885 sendSessionTerminate(Reason.FAILED_APPLICATION);
886 }
887 }
888
889 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
890 final JinglePacket jinglePacket =
891 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
892 send(jinglePacket);
893 }
894
895 public synchronized void rejectContentAdd() {
896 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
897 if (incomingContentAdd == null) {
898 throw new IllegalStateException("No incoming content add");
899 }
900 this.incomingContentAdd = null;
901 updateEndUserState();
902 rejectContentAdd(incomingContentAdd);
903 }
904
905 private void rejectContentAdd(final RtpContentMap contentMap) {
906 final JinglePacket jinglePacket =
907 contentMap
908 .toStub()
909 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
910 Log.d(
911 Config.LOGTAG,
912 id.getAccount().getJid().asBareJid()
913 + ": rejecting content "
914 + ContentAddition.summary(contentMap));
915 send(jinglePacket);
916 }
917
918 private boolean checkForIceRestart(
919 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
920 final RtpContentMap existing = getRemoteContentMap();
921 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
922 final IceUdpTransportInfo.Credentials newCredentials;
923 try {
924 existingCredentials = existing.getCredentials();
925 newCredentials = rtpContentMap.getDistinctCredentials();
926 } catch (final IllegalStateException e) {
927 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
928 return false;
929 }
930 if (existingCredentials.contains(newCredentials)) {
931 return false;
932 }
933 // TODO an alternative approach is to check if we already got an iq result to our
934 // ICE-restart
935 // and if that's the case we are seeing an answer.
936 // This might be more spec compliant but also more error prone potentially
937 final boolean isSignalStateStable =
938 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
939 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
940 final boolean isOffer = rtpContentMap.emptyCandidates();
941 final RtpContentMap restartContentMap;
942 try {
943 if (isOffer) {
944 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
945 restartContentMap =
946 existing.modifiedCredentials(
947 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
948 } else {
949 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
950 Log.d(
951 Config.LOGTAG,
952 "received confirmation of ICE restart"
953 + newCredentials
954 + " peer_setup="
955 + setup);
956 // DTLS setup attribute needs to be rewritten to reflect current peer state
957 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
958 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
959 }
960 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
961 return isOffer;
962 } else {
963 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
964 respondWithTieBreak(jinglePacket);
965 return true;
966 }
967 } catch (final Exception exception) {
968 respondOk(jinglePacket);
969 final Throwable rootCause = Throwables.getRootCause(exception);
970 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
971 // If this happens a termination is already in progress
972 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
973 return true;
974 }
975 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
976 webRTCWrapper.close();
977 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
978 return true;
979 }
980 }
981
982 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
983 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
984 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
985 throw new IllegalStateException("Invalid peer setup");
986 }
987 return peerSetup;
988 }
989
990 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
991 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
992 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
993 }
994 this.peerDtlsSetup = setup;
995 }
996
997 private boolean applyIceRestart(
998 final JinglePacket jinglePacket,
999 final RtpContentMap restartContentMap,
1000 final boolean isOffer)
1001 throws ExecutionException, InterruptedException {
1002 final SessionDescription sessionDescription =
1003 SessionDescription.of(restartContentMap, isResponder());
1004 final org.webrtc.SessionDescription.Type type =
1005 isOffer
1006 ? org.webrtc.SessionDescription.Type.OFFER
1007 : org.webrtc.SessionDescription.Type.ANSWER;
1008 org.webrtc.SessionDescription sdp =
1009 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1010 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1011 if (isInitiator()) {
1012 // We ignore the offer and respond with tie-break. This will clause the responder
1013 // not to apply the content map
1014 return false;
1015 }
1016 }
1017 webRTCWrapper.setRemoteDescription(sdp).get();
1018 setRemoteContentMap(restartContentMap);
1019 if (isOffer) {
1020 final SessionDescription localSessionDescription = setLocalSessionDescription();
1021 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1022 // We need to respond OK before sending any candidates
1023 respondOk(jinglePacket);
1024 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1025 } else {
1026 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1027 }
1028 return true;
1029 }
1030
1031 private void processCandidates(
1032 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1033 contents) {
1034 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1035 content : contents) {
1036 processCandidate(content);
1037 }
1038 }
1039
1040 private void processCandidate(
1041 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1042 content) {
1043 final RtpContentMap rtpContentMap = getRemoteContentMap();
1044 final List<String> indices = toIdentificationTags(rtpContentMap);
1045 final String sdpMid = content.getKey(); // aka content name
1046 final IceUdpTransportInfo transport = content.getValue().transport;
1047 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1048
1049 // TODO check that credentials remained the same
1050
1051 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1052 final String sdp;
1053 try {
1054 sdp = candidate.toSdpAttribute(credentials.ufrag);
1055 } catch (final IllegalArgumentException e) {
1056 Log.d(
1057 Config.LOGTAG,
1058 id.account.getJid().asBareJid()
1059 + ": ignoring invalid ICE candidate "
1060 + e.getMessage());
1061 continue;
1062 }
1063 final int mLineIndex = indices.indexOf(sdpMid);
1064 if (mLineIndex < 0) {
1065 Log.w(
1066 Config.LOGTAG,
1067 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1068 }
1069 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1070 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1071 this.webRTCWrapper.addIceCandidate(iceCandidate);
1072 }
1073 }
1074
1075 private RtpContentMap getRemoteContentMap() {
1076 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1077 }
1078
1079 private RtpContentMap getLocalContentMap() {
1080 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1081 }
1082
1083 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1084 final Group originalGroup = rtpContentMap.group;
1085 final List<String> identificationTags =
1086 originalGroup == null
1087 ? rtpContentMap.getNames()
1088 : originalGroup.getIdentificationTags();
1089 if (identificationTags.size() == 0) {
1090 Log.w(
1091 Config.LOGTAG,
1092 id.account.getJid().asBareJid()
1093 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1094 }
1095 return identificationTags;
1096 }
1097
1098 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1099 final JinglePacket jinglePacket, final boolean expectVerification) {
1100 try {
1101 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1102 } catch (final Exception e) {
1103 return Futures.immediateFailedFuture(e);
1104 }
1105 }
1106
1107 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1108 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1109 Log.d(
1110 Config.LOGTAG,
1111 "receiveRtpContentMap("
1112 + receivedContentMap.getClass().getSimpleName()
1113 + ",expectVerification="
1114 + expectVerification
1115 + ")");
1116 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1117 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1118 id.account
1119 .getAxolotlService()
1120 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1121 return Futures.transform(
1122 future,
1123 omemoVerifiedPayload -> {
1124 // TODO test if an exception here triggers a correct abort
1125 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1126 Log.d(
1127 Config.LOGTAG,
1128 id.account.getJid().asBareJid()
1129 + ": received verifiable DTLS fingerprint via "
1130 + omemoVerification);
1131 return omemoVerifiedPayload.getPayload();
1132 },
1133 MoreExecutors.directExecutor());
1134 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1135 return Futures.immediateFailedFuture(
1136 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1137 } else {
1138 return Futures.immediateFuture(receivedContentMap);
1139 }
1140 }
1141
1142 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1143 if (isInitiator()) {
1144 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1145 return;
1146 }
1147 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1148 Futures.addCallback(
1149 future,
1150 new FutureCallback<>() {
1151 @Override
1152 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1153 receiveSessionInitiate(jinglePacket, rtpContentMap);
1154 }
1155
1156 @Override
1157 public void onFailure(@NonNull final Throwable throwable) {
1158 respondOk(jinglePacket);
1159 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1160 }
1161 },
1162 MoreExecutors.directExecutor());
1163 }
1164
1165 private void receiveSessionInitiate(
1166 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1167 try {
1168 contentMap.requireContentDescriptions();
1169 contentMap.requireDTLSFingerprint(true);
1170 } catch (final RuntimeException e) {
1171 Log.d(
1172 Config.LOGTAG,
1173 id.account.getJid().asBareJid() + ": improperly formatted contents",
1174 Throwables.getRootCause(e));
1175 respondOk(jinglePacket);
1176 sendSessionTerminate(Reason.of(e), e.getMessage());
1177 return;
1178 }
1179 Log.d(
1180 Config.LOGTAG,
1181 "processing session-init with " + contentMap.contents.size() + " contents");
1182 final State target;
1183 if (this.state == State.PROCEED) {
1184 Preconditions.checkState(
1185 proposedMedia != null && proposedMedia.size() > 0,
1186 "proposed media must be set when processing pre-approved session-initiate");
1187 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1188 sendSessionTerminate(
1189 Reason.SECURITY_ERROR,
1190 String.format(
1191 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1192 this.proposedMedia, contentMap.getMedia()));
1193 return;
1194 }
1195 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1196 } else {
1197 target = State.SESSION_INITIALIZED;
1198 setProposedMedia(contentMap.getMedia());
1199 }
1200 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1201 respondOk(jinglePacket);
1202 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1203 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1204 Log.d(
1205 Config.LOGTAG,
1206 id.account.getJid().asBareJid()
1207 + ": automatically accepting session-initiate");
1208 sendSessionAccept();
1209 } else {
1210 Log.d(
1211 Config.LOGTAG,
1212 id.account.getJid().asBareJid()
1213 + ": received not pre-approved session-initiate. start ringing");
1214 startRinging();
1215 }
1216 } else {
1217 Log.d(
1218 Config.LOGTAG,
1219 String.format(
1220 "%s: received session-initiate while in state %s",
1221 id.account.getJid().asBareJid(), state));
1222 terminateWithOutOfOrder(jinglePacket);
1223 }
1224 }
1225
1226 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1227 if (isResponder()) {
1228 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1229 return;
1230 }
1231 final ListenableFuture<RtpContentMap> future =
1232 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1233 Futures.addCallback(
1234 future,
1235 new FutureCallback<>() {
1236 @Override
1237 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1238 receiveSessionAccept(jinglePacket, rtpContentMap);
1239 }
1240
1241 @Override
1242 public void onFailure(@NonNull final Throwable throwable) {
1243 respondOk(jinglePacket);
1244 Log.d(
1245 Config.LOGTAG,
1246 id.account.getJid().asBareJid()
1247 + ": improperly formatted contents in session-accept",
1248 throwable);
1249 webRTCWrapper.close();
1250 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1251 }
1252 },
1253 MoreExecutors.directExecutor());
1254 }
1255
1256 private void receiveSessionAccept(
1257 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1258 try {
1259 contentMap.requireContentDescriptions();
1260 contentMap.requireDTLSFingerprint();
1261 } catch (final RuntimeException e) {
1262 respondOk(jinglePacket);
1263 Log.d(
1264 Config.LOGTAG,
1265 id.account.getJid().asBareJid()
1266 + ": improperly formatted contents in session-accept",
1267 e);
1268 webRTCWrapper.close();
1269 sendSessionTerminate(Reason.of(e), e.getMessage());
1270 return;
1271 }
1272 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1273 if (!initiatorMedia.equals(contentMap.getMedia())) {
1274 sendSessionTerminate(
1275 Reason.SECURITY_ERROR,
1276 String.format(
1277 "Your session-included included media %s but our session-initiate was %s",
1278 this.proposedMedia, contentMap.getMedia()));
1279 return;
1280 }
1281 Log.d(
1282 Config.LOGTAG,
1283 "processing session-accept with " + contentMap.contents.size() + " contents");
1284 if (transition(State.SESSION_ACCEPTED)) {
1285 respondOk(jinglePacket);
1286 receiveSessionAccept(contentMap);
1287 } else {
1288 Log.d(
1289 Config.LOGTAG,
1290 String.format(
1291 "%s: received session-accept while in state %s",
1292 id.account.getJid().asBareJid(), state));
1293 respondOk(jinglePacket);
1294 }
1295 }
1296
1297 private void receiveSessionAccept(final RtpContentMap contentMap) {
1298 this.responderRtpContentMap = contentMap;
1299 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1300 final SessionDescription sessionDescription;
1301 try {
1302 sessionDescription = SessionDescription.of(contentMap, false);
1303 } catch (final IllegalArgumentException | NullPointerException e) {
1304 Log.d(
1305 Config.LOGTAG,
1306 id.account.getJid().asBareJid()
1307 + ": unable convert offer from session-accept to SDP",
1308 e);
1309 webRTCWrapper.close();
1310 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1311 return;
1312 }
1313 final org.webrtc.SessionDescription answer =
1314 new org.webrtc.SessionDescription(
1315 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1316 try {
1317 this.webRTCWrapper.setRemoteDescription(answer).get();
1318 } catch (final Exception e) {
1319 Log.d(
1320 Config.LOGTAG,
1321 id.account.getJid().asBareJid()
1322 + ": unable to set remote description after receiving session-accept",
1323 Throwables.getRootCause(e));
1324 webRTCWrapper.close();
1325 sendSessionTerminate(
1326 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1327 return;
1328 }
1329 processCandidates(contentMap.contents.entrySet());
1330 }
1331
1332 private void sendSessionAccept() {
1333 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1334 if (rtpContentMap == null) {
1335 throw new IllegalStateException("initiator RTP Content Map has not been set");
1336 }
1337 final SessionDescription offer;
1338 try {
1339 offer = SessionDescription.of(rtpContentMap, true);
1340 } catch (final IllegalArgumentException | NullPointerException e) {
1341 Log.d(
1342 Config.LOGTAG,
1343 id.account.getJid().asBareJid()
1344 + ": unable convert offer from session-initiate to SDP",
1345 e);
1346 webRTCWrapper.close();
1347 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1348 return;
1349 }
1350 sendSessionAccept(rtpContentMap.getMedia(), offer);
1351 }
1352
1353 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1354 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1355 }
1356
1357 private synchronized void sendSessionAccept(
1358 final Set<Media> media,
1359 final SessionDescription offer,
1360 final List<PeerConnection.IceServer> iceServers) {
1361 if (isTerminated()) {
1362 Log.w(
1363 Config.LOGTAG,
1364 id.account.getJid().asBareJid()
1365 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1366 return;
1367 }
1368 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1369 try {
1370 setupWebRTC(media, iceServers, !includeCandidates);
1371 } catch (final WebRTCWrapper.InitializationException e) {
1372 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1373 webRTCWrapper.close();
1374 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1375 return;
1376 }
1377 final org.webrtc.SessionDescription sdp =
1378 new org.webrtc.SessionDescription(
1379 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1380 try {
1381 this.webRTCWrapper.setRemoteDescription(sdp).get();
1382 addIceCandidatesFromBlackLog();
1383 org.webrtc.SessionDescription webRTCSessionDescription =
1384 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1385 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1386 } catch (final Exception e) {
1387 failureToAcceptSession(e);
1388 }
1389 }
1390
1391 private void failureToAcceptSession(final Throwable throwable) {
1392 if (isTerminated()) {
1393 return;
1394 }
1395 final Throwable rootCause = Throwables.getRootCause(throwable);
1396 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1397 webRTCWrapper.close();
1398 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1399 }
1400
1401 private void failureToPerformAction(
1402 final JinglePacket.Action action, final Throwable throwable) {
1403 if (isTerminated()) {
1404 return;
1405 }
1406 final Throwable rootCause = Throwables.getRootCause(throwable);
1407 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1408 webRTCWrapper.close();
1409 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1410 }
1411
1412 private void addIceCandidatesFromBlackLog() {
1413 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1414 while ((foo = this.pendingIceCandidates.poll()) != null) {
1415 processCandidate(foo);
1416 Log.d(
1417 Config.LOGTAG,
1418 id.account.getJid().asBareJid() + ": added candidate from back log");
1419 }
1420 }
1421
1422 private void prepareSessionAccept(
1423 final org.webrtc.SessionDescription webRTCSessionDescription,
1424 final boolean includeCandidates) {
1425 final SessionDescription sessionDescription =
1426 SessionDescription.parse(webRTCSessionDescription.description);
1427 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1428 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1429 if (includeCandidates) {
1430 candidates = parseCandidates(sessionDescription);
1431 } else {
1432 candidates = ImmutableMultimap.of();
1433 }
1434 this.responderRtpContentMap = respondingRtpContentMap;
1435 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1436 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1437 prepareOutgoingContentMap(respondingRtpContentMap);
1438 Futures.addCallback(
1439 outgoingContentMapFuture,
1440 new FutureCallback<>() {
1441 @Override
1442 public void onSuccess(final RtpContentMap outgoingContentMap) {
1443 if (includeCandidates) {
1444 Log.d(
1445 Config.LOGTAG,
1446 "including "
1447 + candidates.size()
1448 + " candidates in session accept");
1449 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1450 } else {
1451 sendSessionAccept(outgoingContentMap);
1452 }
1453 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1454 }
1455
1456 @Override
1457 public void onFailure(@NonNull Throwable throwable) {
1458 failureToAcceptSession(throwable);
1459 }
1460 },
1461 MoreExecutors.directExecutor());
1462 }
1463
1464 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1465 if (isTerminated()) {
1466 Log.w(
1467 Config.LOGTAG,
1468 id.account.getJid().asBareJid()
1469 + ": preparing session accept was too slow. already terminated. nothing to do.");
1470 return;
1471 }
1472 transitionOrThrow(State.SESSION_ACCEPTED);
1473 final JinglePacket sessionAccept =
1474 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1475 send(sessionAccept);
1476 }
1477
1478 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1479 final RtpContentMap rtpContentMap) {
1480 if (this.omemoVerification.hasDeviceId()) {
1481 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1482 verifiedPayloadFuture =
1483 id.account
1484 .getAxolotlService()
1485 .encrypt(
1486 rtpContentMap,
1487 id.with,
1488 omemoVerification.getDeviceId());
1489 return Futures.transform(
1490 verifiedPayloadFuture,
1491 verifiedPayload -> {
1492 omemoVerification.setOrEnsureEqual(verifiedPayload);
1493 return verifiedPayload.getPayload();
1494 },
1495 MoreExecutors.directExecutor());
1496 } else {
1497 return Futures.immediateFuture(rtpContentMap);
1498 }
1499 }
1500
1501 synchronized void deliveryMessage(
1502 final Jid from,
1503 final Element message,
1504 final String serverMessageId,
1505 final long timestamp) {
1506 Log.d(
1507 Config.LOGTAG,
1508 id.account.getJid().asBareJid()
1509 + ": delivered message to JingleRtpConnection "
1510 + message);
1511 switch (message.getName()) {
1512 case "propose" -> receivePropose(
1513 from, Propose.upgrade(message), serverMessageId, timestamp);
1514 case "proceed" -> receiveProceed(
1515 from, Proceed.upgrade(message), serverMessageId, timestamp);
1516 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1517 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1518 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1519 }
1520 }
1521
1522 void deliverFailedProceed(final String message) {
1523 Log.d(
1524 Config.LOGTAG,
1525 id.account.getJid().asBareJid()
1526 + ": receive message error for proceed message ("
1527 + Strings.nullToEmpty(message)
1528 + ")");
1529 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1530 webRTCWrapper.close();
1531 Log.d(
1532 Config.LOGTAG,
1533 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1534 this.finish();
1535 }
1536 }
1537
1538 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1539 final boolean originatedFromMyself =
1540 from.asBareJid().equals(id.account.getJid().asBareJid());
1541 if (originatedFromMyself) {
1542 if (transition(State.ACCEPTED)) {
1543 acceptedOnOtherDevice(serverMsgId, timestamp);
1544 } else {
1545 Log.d(
1546 Config.LOGTAG,
1547 id.account.getJid().asBareJid()
1548 + ": unable to transition to accept because already in state="
1549 + this.state);
1550 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1551 }
1552 } else {
1553 Log.d(
1554 Config.LOGTAG,
1555 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1556 }
1557 }
1558
1559 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1560 if (serverMsgId != null) {
1561 this.message.setServerMsgId(serverMsgId);
1562 }
1563 this.message.setTime(timestamp);
1564 this.message.setCarbon(true); // indicate that call was accepted on other device
1565 this.writeLogMessageSuccess(0);
1566 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1567 this.finish();
1568 }
1569
1570 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1571 final boolean originatedFromMyself =
1572 from.asBareJid().equals(id.account.getJid().asBareJid());
1573 // reject from another one of my clients
1574 if (originatedFromMyself) {
1575 receiveRejectFromMyself(serverMsgId, timestamp);
1576 } else if (isInitiator()) {
1577 if (from.equals(id.with)) {
1578 receiveRejectFromResponder();
1579 } else {
1580 Log.d(
1581 Config.LOGTAG,
1582 id.account.getJid()
1583 + ": ignoring reject from "
1584 + from
1585 + " for session with "
1586 + id.with);
1587 }
1588 } else {
1589 Log.d(
1590 Config.LOGTAG,
1591 id.account.getJid()
1592 + ": ignoring reject from "
1593 + from
1594 + " for session with "
1595 + id.with);
1596 }
1597 }
1598
1599 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1600 if (transition(State.REJECTED)) {
1601 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1602 this.finish();
1603 if (serverMsgId != null) {
1604 this.message.setServerMsgId(serverMsgId);
1605 }
1606 this.message.setTime(timestamp);
1607 this.message.setCarbon(true); // indicate that call was rejected on other device
1608 writeLogMessageMissed();
1609 } else {
1610 Log.d(
1611 Config.LOGTAG,
1612 "not able to transition into REJECTED because already in " + this.state);
1613 }
1614 }
1615
1616 private void receiveRejectFromResponder() {
1617 if (isInState(State.PROCEED)) {
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": received reject while still in proceed. callee reconsidered");
1622 closeTransitionLogFinish(State.REJECTED_RACED);
1623 return;
1624 }
1625 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1626 Log.d(
1627 Config.LOGTAG,
1628 id.account.getJid()
1629 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1630 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1631 return;
1632 }
1633 Log.d(
1634 Config.LOGTAG,
1635 id.account.getJid()
1636 + ": ignoring reject from responder because already in state "
1637 + this.state);
1638 }
1639
1640 private void receivePropose(
1641 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1642 final boolean originatedFromMyself =
1643 from.asBareJid().equals(id.account.getJid().asBareJid());
1644 if (originatedFromMyself) {
1645 Log.d(
1646 Config.LOGTAG,
1647 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1648 } else if (transition(
1649 State.PROPOSED,
1650 () -> {
1651 final Collection<RtpDescription> descriptions =
1652 Collections2.transform(
1653 Collections2.filter(
1654 propose.getDescriptions(),
1655 d -> d instanceof RtpDescription),
1656 input -> (RtpDescription) input);
1657 final Collection<Media> media =
1658 Collections2.transform(descriptions, RtpDescription::getMedia);
1659 Preconditions.checkState(
1660 !media.contains(Media.UNKNOWN),
1661 "RTP descriptions contain unknown media");
1662 Log.d(
1663 Config.LOGTAG,
1664 id.account.getJid().asBareJid()
1665 + ": received session proposal from "
1666 + from
1667 + " for "
1668 + media);
1669 this.setProposedMedia(Sets.newHashSet(media));
1670 })) {
1671 if (serverMsgId != null) {
1672 this.message.setServerMsgId(serverMsgId);
1673 }
1674 this.message.setTime(timestamp);
1675 startRinging();
1676 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1677 sendJingleMessage("ringing");
1678 }
1679 } else {
1680 Log.d(
1681 Config.LOGTAG,
1682 id.account.getJid()
1683 + ": ignoring session proposal because already in "
1684 + state);
1685 }
1686 }
1687
1688 private void startRinging() {
1689 this.callIntegration.setRinging();
1690 Log.d(
1691 Config.LOGTAG,
1692 id.account.getJid().asBareJid()
1693 + ": received call from "
1694 + id.with
1695 + ". start ringing");
1696 ringingTimeoutFuture =
1697 jingleConnectionManager.schedule(
1698 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1699 if (CallIntegration.selfManaged(xmppConnectionService)) {
1700 return;
1701 }
1702 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1703 }
1704
1705 private synchronized void ringingTimeout() {
1706 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1707 switch (this.state) {
1708 case PROPOSED -> {
1709 message.markUnread();
1710 rejectCallFromProposed();
1711 }
1712 case SESSION_INITIALIZED -> {
1713 message.markUnread();
1714 rejectCallFromSessionInitiate();
1715 }
1716 }
1717 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1718 }
1719
1720 private void cancelRingingTimeout() {
1721 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1722 if (future != null && !future.isCancelled()) {
1723 future.cancel(false);
1724 }
1725 }
1726
1727 private void receiveProceed(
1728 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1729 final Set<Media> media =
1730 Preconditions.checkNotNull(
1731 this.proposedMedia, "Proposed media has to be set before handling proceed");
1732 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1733 if (from.equals(id.with)) {
1734 if (isInitiator()) {
1735 if (transition(State.PROCEED)) {
1736 if (serverMsgId != null) {
1737 this.message.setServerMsgId(serverMsgId);
1738 }
1739 this.message.setTime(timestamp);
1740 final Integer remoteDeviceId = proceed.getDeviceId();
1741 if (isOmemoEnabled()) {
1742 this.omemoVerification.setDeviceId(remoteDeviceId);
1743 } else {
1744 if (remoteDeviceId != null) {
1745 Log.d(
1746 Config.LOGTAG,
1747 id.account.getJid().asBareJid()
1748 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1749 }
1750 this.omemoVerification.setDeviceId(null);
1751 }
1752 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1753 } else {
1754 Log.d(
1755 Config.LOGTAG,
1756 String.format(
1757 "%s: ignoring proceed because already in %s",
1758 id.account.getJid().asBareJid(), this.state));
1759 }
1760 } else {
1761 Log.d(
1762 Config.LOGTAG,
1763 String.format(
1764 "%s: ignoring proceed because we were not initializing",
1765 id.account.getJid().asBareJid()));
1766 }
1767 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1768 if (transition(State.ACCEPTED)) {
1769 Log.d(
1770 Config.LOGTAG,
1771 id.account.getJid().asBareJid()
1772 + ": moved session with "
1773 + id.with
1774 + " into state accepted after received carbon copied proceed");
1775 acceptedOnOtherDevice(serverMsgId, timestamp);
1776 }
1777 } else {
1778 Log.d(
1779 Config.LOGTAG,
1780 String.format(
1781 "%s: ignoring proceed from %s. was expected from %s",
1782 id.account.getJid().asBareJid(), from, id.with));
1783 }
1784 }
1785
1786 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1787 if (from.equals(id.with)) {
1788 final State target =
1789 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1790 if (transition(target)) {
1791 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1792 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1793 Log.d(
1794 Config.LOGTAG,
1795 id.account.getJid().asBareJid()
1796 + ": session with "
1797 + id.with
1798 + " has been retracted (serverMsgId="
1799 + serverMsgId
1800 + ")");
1801 if (serverMsgId != null) {
1802 this.message.setServerMsgId(serverMsgId);
1803 }
1804 this.message.setTime(timestamp);
1805 if (target == State.RETRACTED) {
1806 this.message.markUnread();
1807 }
1808 writeLogMessageMissed();
1809 finish();
1810 } else {
1811 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1812 }
1813 } else {
1814 // TODO parse retract from self
1815 Log.d(
1816 Config.LOGTAG,
1817 id.account.getJid().asBareJid()
1818 + ": received retract from "
1819 + from
1820 + ". expected retract from"
1821 + id.with
1822 + ". ignoring");
1823 }
1824 }
1825
1826 public void sendSessionInitiate() {
1827 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1828 }
1829
1830 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1831 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1832 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1833 }
1834
1835 private synchronized void sendSessionInitiate(
1836 final Set<Media> media,
1837 final State targetState,
1838 final List<PeerConnection.IceServer> iceServers) {
1839 if (isTerminated()) {
1840 Log.w(
1841 Config.LOGTAG,
1842 id.account.getJid().asBareJid()
1843 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1844 return;
1845 }
1846 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1847 try {
1848 setupWebRTC(media, iceServers, !includeCandidates);
1849 } catch (final WebRTCWrapper.InitializationException e) {
1850 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1851 webRTCWrapper.close();
1852 sendRetract(Reason.ofThrowable(e));
1853 return;
1854 }
1855 try {
1856 org.webrtc.SessionDescription webRTCSessionDescription =
1857 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1858 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1859 } catch (final Exception e) {
1860 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1861 // exceptions
1862 failureToInitiateSession(e, targetState);
1863 }
1864 }
1865
1866 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1867 if (isTerminated()) {
1868 return;
1869 }
1870 Log.d(
1871 Config.LOGTAG,
1872 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1873 Throwables.getRootCause(throwable));
1874 webRTCWrapper.close();
1875 final Reason reason = Reason.ofThrowable(throwable);
1876 if (isInState(targetState)) {
1877 sendSessionTerminate(reason, throwable.getMessage());
1878 } else {
1879 sendRetract(reason);
1880 }
1881 }
1882
1883 private void sendRetract(final Reason reason) {
1884 // TODO embed reason into retract
1885 sendJingleMessage("retract", id.with.asBareJid());
1886 transitionOrThrow(reasonToState(reason));
1887 this.finish();
1888 }
1889
1890 private void prepareSessionInitiate(
1891 final org.webrtc.SessionDescription webRTCSessionDescription,
1892 final boolean includeCandidates,
1893 final State targetState) {
1894 final SessionDescription sessionDescription =
1895 SessionDescription.parse(webRTCSessionDescription.description);
1896 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1897 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1898 if (includeCandidates) {
1899 candidates = parseCandidates(sessionDescription);
1900 } else {
1901 candidates = ImmutableMultimap.of();
1902 }
1903 this.initiatorRtpContentMap = rtpContentMap;
1904 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1905 encryptSessionInitiate(rtpContentMap);
1906 Futures.addCallback(
1907 outgoingContentMapFuture,
1908 new FutureCallback<>() {
1909 @Override
1910 public void onSuccess(final RtpContentMap outgoingContentMap) {
1911 if (includeCandidates) {
1912 Log.d(
1913 Config.LOGTAG,
1914 "including "
1915 + candidates.size()
1916 + " candidates in session initiate");
1917 sendSessionInitiate(
1918 outgoingContentMap.withCandidates(candidates), targetState);
1919 } else {
1920 sendSessionInitiate(outgoingContentMap, targetState);
1921 }
1922 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1923 }
1924
1925 @Override
1926 public void onFailure(@NonNull final Throwable throwable) {
1927 failureToInitiateSession(throwable, targetState);
1928 }
1929 },
1930 MoreExecutors.directExecutor());
1931 }
1932
1933 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1934 if (isTerminated()) {
1935 Log.w(
1936 Config.LOGTAG,
1937 id.account.getJid().asBareJid()
1938 + ": preparing session was too slow. already terminated. nothing to do.");
1939 return;
1940 }
1941 this.transitionOrThrow(targetState);
1942 final JinglePacket sessionInitiate =
1943 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1944 send(sessionInitiate);
1945 }
1946
1947 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1948 final RtpContentMap rtpContentMap) {
1949 if (this.omemoVerification.hasDeviceId()) {
1950 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1951 verifiedPayloadFuture =
1952 id.account
1953 .getAxolotlService()
1954 .encrypt(
1955 rtpContentMap,
1956 id.with,
1957 omemoVerification.getDeviceId());
1958 final ListenableFuture<RtpContentMap> future =
1959 Futures.transform(
1960 verifiedPayloadFuture,
1961 verifiedPayload -> {
1962 omemoVerification.setSessionFingerprint(
1963 verifiedPayload.getFingerprint());
1964 return verifiedPayload.getPayload();
1965 },
1966 MoreExecutors.directExecutor());
1967 if (Config.REQUIRE_RTP_VERIFICATION) {
1968 return future;
1969 }
1970 return Futures.catching(
1971 future,
1972 CryptoFailedException.class,
1973 e -> {
1974 Log.w(
1975 Config.LOGTAG,
1976 id.account.getJid().asBareJid()
1977 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1978 e);
1979 return rtpContentMap;
1980 },
1981 MoreExecutors.directExecutor());
1982 } else {
1983 return Futures.immediateFuture(rtpContentMap);
1984 }
1985 }
1986
1987 protected void sendSessionTerminate(final Reason reason) {
1988 sendSessionTerminate(reason, null);
1989 }
1990
1991 protected void sendSessionTerminate(final Reason reason, final String text) {
1992 sendSessionTerminate(reason, text, this::writeLogMessage);
1993 sendJingleMessageFinish(reason);
1994 }
1995
1996 private void sendTransportInfo(
1997 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1998 final RtpContentMap transportInfo;
1999 try {
2000 final RtpContentMap rtpContentMap =
2001 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2002 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2003 } catch (final Exception e) {
2004 Log.d(
2005 Config.LOGTAG,
2006 id.account.getJid().asBareJid()
2007 + ": unable to prepare transport-info from candidate for content="
2008 + contentName);
2009 return;
2010 }
2011 final JinglePacket jinglePacket =
2012 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2013 send(jinglePacket);
2014 }
2015
2016 public RtpEndUserState getEndUserState() {
2017 switch (this.state) {
2018 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2019 if (isInitiator()) {
2020 return RtpEndUserState.RINGING;
2021 } else {
2022 return RtpEndUserState.INCOMING_CALL;
2023 }
2024 }
2025 case PROCEED -> {
2026 if (isInitiator()) {
2027 return RtpEndUserState.RINGING;
2028 } else {
2029 return RtpEndUserState.ACCEPTING_CALL;
2030 }
2031 }
2032 case SESSION_INITIALIZED_PRE_APPROVED -> {
2033 if (isInitiator()) {
2034 return RtpEndUserState.RINGING;
2035 } else {
2036 return RtpEndUserState.CONNECTING;
2037 }
2038 }
2039 case SESSION_ACCEPTED -> {
2040 final ContentAddition ca = getPendingContentAddition();
2041 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2042 return RtpEndUserState.INCOMING_CONTENT_ADD;
2043 }
2044 return getPeerConnectionStateAsEndUserState();
2045 }
2046 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2047 if (isInitiator()) {
2048 return RtpEndUserState.DECLINED_OR_BUSY;
2049 } else {
2050 return RtpEndUserState.ENDED;
2051 }
2052 }
2053 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2054 return RtpEndUserState.ENDED;
2055 }
2056 case RETRACTED_RACED -> {
2057 if (isInitiator()) {
2058 return RtpEndUserState.ENDED;
2059 } else {
2060 return RtpEndUserState.RETRACTED;
2061 }
2062 }
2063 case TERMINATED_CONNECTIVITY_ERROR -> {
2064 return zeroDuration()
2065 ? RtpEndUserState.CONNECTIVITY_ERROR
2066 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2067 }
2068 case TERMINATED_APPLICATION_FAILURE -> {
2069 return RtpEndUserState.APPLICATION_ERROR;
2070 }
2071 case TERMINATED_SECURITY_ERROR -> {
2072 return RtpEndUserState.SECURITY_ERROR;
2073 }
2074 }
2075 throw new IllegalStateException(
2076 String.format("%s has no equivalent EndUserState", this.state));
2077 }
2078
2079 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2080 final PeerConnection.PeerConnectionState state;
2081 try {
2082 state = webRTCWrapper.getState();
2083 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2084 // We usually close the WebRTCWrapper *before* transitioning so we might still
2085 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2086 return RtpEndUserState.ENDING_CALL;
2087 }
2088 return switch (state) {
2089 case CONNECTED -> RtpEndUserState.CONNECTED;
2090 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2091 case CLOSED -> RtpEndUserState.ENDING_CALL;
2092 default -> zeroDuration()
2093 ? RtpEndUserState.CONNECTIVITY_ERROR
2094 : RtpEndUserState.RECONNECTING;
2095 };
2096 }
2097
2098 private boolean isPeerConnectionConnected() {
2099 try {
2100 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2101 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2102 return false;
2103 }
2104 }
2105
2106 private void updateCallIntegrationState() {
2107 switch (this.state) {
2108 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2109 if (isInitiator()) {
2110 this.callIntegration.setDialing();
2111 } else {
2112 this.callIntegration.setRinging();
2113 }
2114 }
2115 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2116 if (isInitiator()) {
2117 this.callIntegration.setDialing();
2118 } else {
2119 this.callIntegration.setInitialized();
2120 }
2121 }
2122 case SESSION_ACCEPTED -> {
2123 if (isPeerConnectionConnected()) {
2124 this.callIntegration.setActive();
2125 } else {
2126 this.callIntegration.setInitialized();
2127 }
2128 }
2129 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2130 if (isInitiator()) {
2131 this.callIntegration.busy();
2132 } else {
2133 this.callIntegration.rejected();
2134 }
2135 }
2136 case TERMINATED_SUCCESS -> this.callIntegration.success();
2137 case ACCEPTED -> this.callIntegration.accepted();
2138 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2139 .retracted();
2140 case TERMINATED_CONNECTIVITY_ERROR,
2141 TERMINATED_APPLICATION_FAILURE,
2142 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2143 default -> throw new IllegalStateException(
2144 String.format("%s is not handled", this.state));
2145 }
2146 }
2147
2148 public ContentAddition getPendingContentAddition() {
2149 final RtpContentMap in = this.incomingContentAdd;
2150 final RtpContentMap out = this.outgoingContentAdd;
2151 if (out != null) {
2152 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2153 } else if (in != null) {
2154 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2155 } else {
2156 return null;
2157 }
2158 }
2159
2160 public Set<Media> getMedia() {
2161 final State current = getState();
2162 if (current == State.NULL) {
2163 if (isInitiator()) {
2164 return Preconditions.checkNotNull(
2165 this.proposedMedia, "RTP connection has not been initialized properly");
2166 }
2167 throw new IllegalStateException("RTP connection has not been initialized yet");
2168 }
2169 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2170 return Preconditions.checkNotNull(
2171 this.proposedMedia, "RTP connection has not been initialized properly");
2172 }
2173 final RtpContentMap localContentMap = getLocalContentMap();
2174 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2175 if (localContentMap != null) {
2176 return localContentMap.getMedia();
2177 } else if (initiatorContentMap != null) {
2178 return initiatorContentMap.getMedia();
2179 } else if (isTerminated()) {
2180 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2181 } else {
2182 return Preconditions.checkNotNull(
2183 this.proposedMedia, "RTP connection has not been initialized properly");
2184 }
2185 }
2186
2187 public boolean isVerified() {
2188 final String fingerprint = this.omemoVerification.getFingerprint();
2189 if (fingerprint == null) {
2190 return false;
2191 }
2192 final FingerprintStatus status =
2193 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2194 return status != null && status.isVerified();
2195 }
2196
2197 public boolean addMedia(final Media media) {
2198 final Set<Media> currentMedia = getMedia();
2199 if (currentMedia.contains(media)) {
2200 throw new IllegalStateException(String.format("%s has already been proposed", media));
2201 }
2202 // TODO add state protection - can only add while ACCEPTED or so
2203 Log.d(Config.LOGTAG, "adding media: " + media);
2204 return webRTCWrapper.addTrack(media);
2205 }
2206
2207 public synchronized void acceptCall() {
2208 switch (this.state) {
2209 case PROPOSED -> {
2210 cancelRingingTimeout();
2211 acceptCallFromProposed();
2212 }
2213 case SESSION_INITIALIZED -> {
2214 cancelRingingTimeout();
2215 acceptCallFromSessionInitialized();
2216 }
2217 case ACCEPTED -> Log.w(
2218 Config.LOGTAG,
2219 id.account.getJid().asBareJid()
2220 + ": the call has already been accepted with another client. UI was just lagging behind");
2221 case PROCEED, SESSION_ACCEPTED -> Log.w(
2222 Config.LOGTAG,
2223 id.account.getJid().asBareJid()
2224 + ": the call has already been accepted. user probably double tapped the UI");
2225 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2226 }
2227 }
2228
2229 public synchronized void rejectCall() {
2230 if (isTerminated()) {
2231 Log.w(
2232 Config.LOGTAG,
2233 id.account.getJid().asBareJid()
2234 + ": received rejectCall() when session has already been terminated. nothing to do");
2235 return;
2236 }
2237 switch (this.state) {
2238 case PROPOSED -> rejectCallFromProposed();
2239 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2240 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2241 }
2242 }
2243
2244 public synchronized void integrationFailure() {
2245 final var state = getState();
2246 if (state == State.PROPOSED) {
2247 Log.e(
2248 Config.LOGTAG,
2249 id.account.getJid().asBareJid()
2250 + ": failed call integration in state proposed");
2251 rejectCallFromProposed();
2252 } else if (state == State.SESSION_INITIALIZED) {
2253 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2254 this.webRTCWrapper.close();
2255 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2256 } else {
2257 throw new IllegalStateException(
2258 String.format("Can not fail integration in state %s", state));
2259 }
2260 }
2261
2262 public synchronized void endCall() {
2263 if (isTerminated()) {
2264 Log.w(
2265 Config.LOGTAG,
2266 id.account.getJid().asBareJid()
2267 + ": received endCall() when session has already been terminated. nothing to do");
2268 return;
2269 }
2270 if (isInState(State.PROPOSED) && isResponder()) {
2271 rejectCallFromProposed();
2272 return;
2273 }
2274 if (isInState(State.PROCEED)) {
2275 if (isInitiator()) {
2276 retractFromProceed();
2277 } else {
2278 rejectCallFromProceed();
2279 }
2280 return;
2281 }
2282 if (isInitiator()
2283 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2284 this.webRTCWrapper.close();
2285 sendSessionTerminate(Reason.CANCEL);
2286 return;
2287 }
2288 if (isInState(State.SESSION_INITIALIZED)) {
2289 rejectCallFromSessionInitiate();
2290 return;
2291 }
2292 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2293 this.webRTCWrapper.close();
2294 sendSessionTerminate(Reason.SUCCESS);
2295 return;
2296 }
2297 if (isInState(
2298 State.TERMINATED_APPLICATION_FAILURE,
2299 State.TERMINATED_CONNECTIVITY_ERROR,
2300 State.TERMINATED_DECLINED_OR_BUSY)) {
2301 Log.d(
2302 Config.LOGTAG,
2303 "ignoring request to end call because already in state " + this.state);
2304 return;
2305 }
2306 throw new IllegalStateException(
2307 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2308 }
2309
2310 private void retractFromProceed() {
2311 Log.d(Config.LOGTAG, "retract from proceed");
2312 this.sendJingleMessage("retract");
2313 closeTransitionLogFinish(State.RETRACTED_RACED);
2314 }
2315
2316 private void closeTransitionLogFinish(final State state) {
2317 this.webRTCWrapper.close();
2318 transitionOrThrow(state);
2319 writeLogMessage(state);
2320 finish();
2321 }
2322
2323 private void setupWebRTC(
2324 final Set<Media> media,
2325 final List<PeerConnection.IceServer> iceServers,
2326 final boolean trickle)
2327 throws WebRTCWrapper.InitializationException {
2328 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2329 this.webRTCWrapper.setup(this.xmppConnectionService);
2330 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2331 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2332 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2333 }
2334
2335 private void acceptCallFromProposed() {
2336 transitionOrThrow(State.PROCEED);
2337 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2338 this.callIntegration.startAudioRouting();
2339 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2340 this.sendJingleMessage("proceed");
2341 }
2342
2343 private void rejectCallFromProposed() {
2344 transitionOrThrow(State.REJECTED);
2345 writeLogMessageMissed();
2346 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2347 this.sendJingleMessage("reject");
2348 finish();
2349 }
2350
2351 private void rejectCallFromProceed() {
2352 this.sendJingleMessage("reject");
2353 closeTransitionLogFinish(State.REJECTED_RACED);
2354 }
2355
2356 private void rejectCallFromSessionInitiate() {
2357 webRTCWrapper.close();
2358 sendSessionTerminate(Reason.DECLINE);
2359 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2360 }
2361
2362 private void sendJingleMessage(final String action) {
2363 sendJingleMessage(action, id.with);
2364 }
2365
2366 private void sendJingleMessage(final String action, final Jid to) {
2367 final MessagePacket messagePacket = new MessagePacket();
2368 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2369 messagePacket.setTo(to);
2370 final Element intent =
2371 messagePacket
2372 .addChild(action, Namespace.JINGLE_MESSAGE)
2373 .setAttribute("id", id.sessionId);
2374 if ("proceed".equals(action)) {
2375 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2376 if (isOmemoEnabled()) {
2377 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2378 final Element device =
2379 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2380 device.setAttribute("id", deviceId);
2381 }
2382 }
2383 messagePacket.addChild("store", "urn:xmpp:hints");
2384 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2385 }
2386
2387 private void sendJingleMessageFinish(final Reason reason) {
2388 final var account = id.getAccount();
2389 final MessagePacket messagePacket =
2390 xmppConnectionService
2391 .getMessageGenerator()
2392 .sessionFinish(id.with, id.sessionId, reason);
2393 xmppConnectionService.sendMessagePacket(account, messagePacket);
2394 }
2395
2396 private boolean isOmemoEnabled() {
2397 final Conversational conversational = message.getConversation();
2398 if (conversational instanceof Conversation) {
2399 return ((Conversation) conversational).getNextEncryption()
2400 == Message.ENCRYPTION_AXOLOTL;
2401 }
2402 return false;
2403 }
2404
2405 private void acceptCallFromSessionInitialized() {
2406 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2407 this.callIntegration.startAudioRouting();
2408 sendSessionAccept();
2409 }
2410
2411 @Override
2412 protected synchronized boolean transition(final State target, final Runnable runnable) {
2413 if (super.transition(target, runnable)) {
2414 updateEndUserState();
2415 updateOngoingCallNotification();
2416 return true;
2417 } else {
2418 return false;
2419 }
2420 }
2421
2422 @Override
2423 public void onIceCandidate(final IceCandidate iceCandidate) {
2424 final RtpContentMap rtpContentMap =
2425 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2426 final IceUdpTransportInfo.Credentials credentials;
2427 try {
2428 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2429 } catch (final IllegalArgumentException e) {
2430 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2431 return;
2432 }
2433 final String uFrag = credentials.ufrag;
2434 final IceUdpTransportInfo.Candidate candidate =
2435 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2436 if (candidate == null) {
2437 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2438 return;
2439 }
2440 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2441 sendTransportInfo(iceCandidate.sdpMid, candidate);
2442 }
2443
2444 @Override
2445 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2446 Log.d(
2447 Config.LOGTAG,
2448 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2449 this.stateHistory.add(newState);
2450 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2451 this.sessionDuration.start();
2452 updateOngoingCallNotification();
2453 } else if (this.sessionDuration.isRunning()) {
2454 this.sessionDuration.stop();
2455 updateOngoingCallNotification();
2456 }
2457
2458 final boolean neverConnected =
2459 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2460
2461 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2462 if (neverConnected) {
2463 if (isTerminated()) {
2464 Log.d(
2465 Config.LOGTAG,
2466 id.account.getJid().asBareJid()
2467 + ": not sending session-terminate after connectivity error because session is already in state "
2468 + this.state);
2469 return;
2470 }
2471 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2472 return;
2473 } else {
2474 this.restartIce();
2475 }
2476 }
2477 updateEndUserState();
2478 }
2479
2480 private void restartIce() {
2481 this.stateHistory.clear();
2482 this.webRTCWrapper.restartIceAsync();
2483 }
2484
2485 @Override
2486 public void onRenegotiationNeeded() {
2487 this.webRTCWrapper.execute(this::renegotiate);
2488 }
2489
2490 private synchronized void renegotiate() {
2491 final SessionDescription sessionDescription;
2492 try {
2493 sessionDescription = setLocalSessionDescription();
2494 } catch (final Exception e) {
2495 final Throwable cause = Throwables.getRootCause(e);
2496 webRTCWrapper.close();
2497 if (isTerminated()) {
2498 Log.d(
2499 Config.LOGTAG,
2500 "failed to renegotiate. session was already terminated",
2501 cause);
2502 return;
2503 }
2504 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2505 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2506 return;
2507 }
2508 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2509 final RtpContentMap currentContentMap = getLocalContentMap();
2510 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2511 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2512
2513 Log.d(
2514 Config.LOGTAG,
2515 id.getAccount().getJid().asBareJid()
2516 + ": renegotiate. iceRestart="
2517 + iceRestart
2518 + " content id diff="
2519 + diff);
2520
2521 if (diff.hasModifications() && iceRestart) {
2522 webRTCWrapper.close();
2523 sendSessionTerminate(
2524 Reason.FAILED_APPLICATION,
2525 "WebRTC unexpectedly tried to modify content and transport at once");
2526 return;
2527 }
2528
2529 if (iceRestart) {
2530 initiateIceRestart(rtpContentMap);
2531 return;
2532 } else if (diff.isEmpty()) {
2533 Log.d(
2534 Config.LOGTAG,
2535 "renegotiation. nothing to do. SignalingState="
2536 + this.webRTCWrapper.getSignalingState());
2537 }
2538
2539 if (diff.added.isEmpty()) {
2540 return;
2541 }
2542 modifyLocalContentMap(rtpContentMap);
2543 sendContentAdd(rtpContentMap, diff.added);
2544 }
2545
2546 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2547 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2548 final JinglePacket jinglePacket =
2549 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2550 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2551 jinglePacket.setTo(id.with);
2552 xmppConnectionService.sendIqPacket(
2553 id.account,
2554 jinglePacket,
2555 (account, response) -> {
2556 if (response.getType() == IqPacket.TYPE.RESULT) {
2557 Log.d(Config.LOGTAG, "received success to our ice restart");
2558 setLocalContentMap(rtpContentMap);
2559 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2560 return;
2561 }
2562 if (response.getType() == IqPacket.TYPE.ERROR) {
2563 if (isTieBreak(response)) {
2564 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2565 return;
2566 }
2567 handleIqErrorResponse(response);
2568 }
2569 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2570 handleIqTimeoutResponse(response);
2571 }
2572 });
2573 }
2574
2575 private boolean isTieBreak(final IqPacket response) {
2576 final Element error = response.findChild("error");
2577 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2578 }
2579
2580 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2581 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2582 this.outgoingContentAdd = contentAdd;
2583 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2584 prepareOutgoingContentMap(contentAdd);
2585 Futures.addCallback(
2586 outgoingContentMapFuture,
2587 new FutureCallback<>() {
2588 @Override
2589 public void onSuccess(final RtpContentMap outgoingContentMap) {
2590 sendContentAdd(outgoingContentMap);
2591 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2592 }
2593
2594 @Override
2595 public void onFailure(@NonNull Throwable throwable) {
2596 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2597 }
2598 },
2599 MoreExecutors.directExecutor());
2600 }
2601
2602 private void sendContentAdd(final RtpContentMap contentAdd) {
2603
2604 final JinglePacket jinglePacket =
2605 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2606 jinglePacket.setTo(id.with);
2607 xmppConnectionService.sendIqPacket(
2608 id.account,
2609 jinglePacket,
2610 (connection, response) -> {
2611 if (response.getType() == IqPacket.TYPE.RESULT) {
2612 Log.d(
2613 Config.LOGTAG,
2614 id.getAccount().getJid().asBareJid()
2615 + ": received ACK to our content-add");
2616 return;
2617 }
2618 if (response.getType() == IqPacket.TYPE.ERROR) {
2619 if (isTieBreak(response)) {
2620 this.outgoingContentAdd = null;
2621 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2622 return;
2623 }
2624 handleIqErrorResponse(response);
2625 }
2626 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2627 handleIqTimeoutResponse(response);
2628 }
2629 });
2630 }
2631
2632 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2633 if (isInitiator()) {
2634 this.initiatorRtpContentMap = rtpContentMap;
2635 } else {
2636 this.responderRtpContentMap = rtpContentMap;
2637 }
2638 }
2639
2640 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2641 if (isInitiator()) {
2642 this.responderRtpContentMap = rtpContentMap;
2643 } else {
2644 this.initiatorRtpContentMap = rtpContentMap;
2645 }
2646 }
2647
2648 // this method is to be used for content map modifications that modify media
2649 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2650 final RtpContentMap activeContents = rtpContentMap.activeContents();
2651 setLocalContentMap(activeContents);
2652 this.callIntegration.setAudioDeviceWhenAvailable(
2653 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2654 updateEndUserState();
2655 }
2656
2657 private SessionDescription setLocalSessionDescription()
2658 throws ExecutionException, InterruptedException {
2659 final org.webrtc.SessionDescription sessionDescription =
2660 this.webRTCWrapper.setLocalDescription(false).get();
2661 return SessionDescription.parse(sessionDescription.description);
2662 }
2663
2664 private void closeWebRTCSessionAfterFailedConnection() {
2665 this.webRTCWrapper.close();
2666 synchronized (this) {
2667 if (isTerminated()) {
2668 Log.d(
2669 Config.LOGTAG,
2670 id.account.getJid().asBareJid()
2671 + ": no need to send session-terminate after failed connection. Other party already did");
2672 return;
2673 }
2674 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2675 }
2676 }
2677
2678 public boolean zeroDuration() {
2679 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2680 }
2681
2682 public long getCallDuration() {
2683 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2684 }
2685
2686 @Override
2687 public CallIntegration getCallIntegration() {
2688 return this.callIntegration;
2689 }
2690
2691 public boolean isMicrophoneEnabled() {
2692 return webRTCWrapper.isMicrophoneEnabled();
2693 }
2694
2695 public boolean setMicrophoneEnabled(final boolean enabled) {
2696 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2697 }
2698
2699 public boolean isVideoEnabled() {
2700 return webRTCWrapper.isVideoEnabled();
2701 }
2702
2703 public void setVideoEnabled(final boolean enabled) {
2704 webRTCWrapper.setVideoEnabled(enabled);
2705 }
2706
2707 public boolean isCameraSwitchable() {
2708 return webRTCWrapper.isCameraSwitchable();
2709 }
2710
2711 public boolean isFrontCamera() {
2712 return webRTCWrapper.isFrontCamera();
2713 }
2714
2715 public ListenableFuture<Boolean> switchCamera() {
2716 return webRTCWrapper.switchCamera();
2717 }
2718
2719 @Override
2720 public synchronized void onCallIntegrationShowIncomingCallUi() {
2721 if (isTerminated()) {
2722 // there might be race conditions with the call integration service invoking this
2723 // callback when the rtp session has already ended.
2724 Log.w(
2725 Config.LOGTAG,
2726 "CallIntegration requested incoming call UI but session was already terminated");
2727 return;
2728 }
2729 // TODO apparently this can be called too early as well?
2730 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2731 }
2732
2733 @Override
2734 public void onCallIntegrationDisconnect() {
2735 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2736 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2737 rejectCall();
2738 } else {
2739 endCall();
2740 }
2741 }
2742
2743 @Override
2744 public void onCallIntegrationReject() {
2745 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2746 try {
2747 rejectCall();
2748 } catch (final IllegalStateException e) {
2749 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2750 }
2751 }
2752
2753 @Override
2754 public void onCallIntegrationAnswer() {
2755 // we need to start the UI to a) show it and b) be able to ask for permissions
2756 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2757 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2758 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2759 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2760 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2761 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2762 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2763 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2764 xmppConnectionService.startActivity(intent);
2765 }
2766
2767 @Override
2768 public void onCallIntegrationSilence() {
2769 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2770 }
2771
2772 @Override
2773 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2774 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2775 // that was previous disabled by the user. A proper implementation would probably be to
2776 // track user choice and enable the microphone with a userEnabled() &&
2777 // callIntegration.isMicrophoneEnabled() condition
2778 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2779 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2780 }
2781
2782 @Override
2783 public void onAudioDeviceChanged(
2784 final CallIntegration.AudioDevice selectedAudioDevice,
2785 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2786 Log.d(
2787 Config.LOGTAG,
2788 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2789 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2790 selectedAudioDevice, availableAudioDevices);
2791 }
2792
2793 private void updateEndUserState() {
2794 final RtpEndUserState endUserState = getEndUserState();
2795 this.updateCallIntegrationState();
2796 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2797 id.account, id.with, id.sessionId, endUserState);
2798 }
2799
2800 private void updateOngoingCallNotification() {
2801 final State state = this.state;
2802 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2803 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2804 && isResponder()) {
2805 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2806 xmppConnectionService.removeOngoingCall();
2807 return;
2808 }
2809 final boolean reconnecting;
2810 if (state == State.SESSION_ACCEPTED) {
2811 reconnecting =
2812 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2813 } else {
2814 reconnecting = false;
2815 }
2816 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2817 } else {
2818 xmppConnectionService.removeOngoingCall();
2819 }
2820 }
2821
2822 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2823 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2824 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2825 request.setTo(id.account.getDomain());
2826 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2827 xmppConnectionService.sendIqPacket(
2828 id.account,
2829 request,
2830 (account, response) -> {
2831 final var iceServers = IceServers.parse(response);
2832 if (iceServers.isEmpty()) {
2833 Log.w(
2834 Config.LOGTAG,
2835 id.account.getJid().asBareJid()
2836 + ": no ICE server found "
2837 + response);
2838 }
2839 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2840 });
2841 } else {
2842 Log.w(
2843 Config.LOGTAG,
2844 id.account.getJid().asBareJid() + ": has no external service discovery");
2845 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2846 }
2847 }
2848
2849 @Override
2850 protected void terminateTransport() {
2851 this.webRTCWrapper.close();
2852 }
2853
2854 @Override
2855 protected void finish() {
2856 if (isTerminated()) {
2857 this.cancelRingingTimeout();
2858 this.callIntegration.verifyDisconnected();
2859 this.webRTCWrapper.verifyClosed();
2860 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2861 super.finish();
2862 } else {
2863 throw new IllegalStateException(
2864 String.format("Unable to call finish from %s", this.state));
2865 }
2866 }
2867
2868 private void writeLogMessage(final State state) {
2869 final long duration = getCallDuration();
2870 if (state == State.TERMINATED_SUCCESS
2871 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2872 writeLogMessageSuccess(duration);
2873 } else {
2874 writeLogMessageMissed();
2875 }
2876 }
2877
2878 private void writeLogMessageSuccess(final long duration) {
2879 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2880 this.writeMessage();
2881 }
2882
2883 private void writeLogMessageMissed() {
2884 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2885 this.writeMessage();
2886 }
2887
2888 private void writeMessage() {
2889 final Conversational conversational = message.getConversation();
2890 if (conversational instanceof Conversation) {
2891 ((Conversation) conversational).add(this.message);
2892 xmppConnectionService.createMessageAsync(message);
2893 xmppConnectionService.updateConversationUi();
2894 } else {
2895 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2896 }
2897 }
2898
2899 public Optional<VideoTrack> getLocalVideoTrack() {
2900 return webRTCWrapper.getLocalVideoTrack();
2901 }
2902
2903 public Optional<VideoTrack> getRemoteVideoTrack() {
2904 return webRTCWrapper.getRemoteVideoTrack();
2905 }
2906
2907 public EglBase.Context getEglBaseContext() {
2908 return webRTCWrapper.getEglBaseContext();
2909 }
2910
2911 void setProposedMedia(final Set<Media> media) {
2912 this.proposedMedia = media;
2913 this.callIntegration.setVideoState(
2914 Media.audioOnly(media)
2915 ? VideoProfile.STATE_AUDIO_ONLY
2916 : VideoProfile.STATE_BIDIRECTIONAL);
2917 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2918 }
2919
2920 public void fireStateUpdate() {
2921 final RtpEndUserState endUserState = getEndUserState();
2922 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2923 id.account, id.with, id.sessionId, endUserState);
2924 }
2925
2926 public boolean isSwitchToVideoAvailable() {
2927 final boolean prerequisite =
2928 Media.audioOnly(getMedia())
2929 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2930 .contains(getEndUserState());
2931 return prerequisite && remoteHasVideoFeature();
2932 }
2933
2934 private boolean remoteHasVideoFeature() {
2935 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2936 }
2937
2938 private boolean remoteHasSdpOfferAnswer() {
2939 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2940 }
2941
2942 @Override
2943 public Account getAccount() {
2944 return id.account;
2945 }
2946
2947 @Override
2948 public Jid getWith() {
2949 return id.with;
2950 }
2951
2952 @Override
2953 public String getSessionId() {
2954 return id.sessionId;
2955 }
2956
2957 private interface OnIceServersDiscovered {
2958 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2959 }
2960}