1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
50import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
51import eu.siacs.conversations.xmpp.stanzas.IqPacket;
52import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROPOSED,
78 State.PROCEED,
79 State.SESSION_INITIALIZED_PRE_APPROVED,
80 State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82
83 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
84 private final Queue<
85 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
86 pendingIceCandidates = new LinkedList<>();
87 private final OmemoVerification omemoVerification = new OmemoVerification();
88 private final CallIntegration callIntegration;
89 private final Message message;
90
91 private Set<Media> proposedMedia;
92 private RtpContentMap initiatorRtpContentMap;
93 private RtpContentMap responderRtpContentMap;
94 private RtpContentMap incomingContentAdd;
95 private RtpContentMap outgoingContentAdd;
96 private IceUdpTransportInfo.Setup peerDtlsSetup;
97 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
98 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
99 private ScheduledFuture<?> ringingTimeoutFuture;
100
101 JingleRtpConnection(
102 final JingleConnectionManager jingleConnectionManager,
103 final Id id,
104 final Jid initiator) {
105 this(
106 jingleConnectionManager,
107 id,
108 initiator,
109 new CallIntegration(
110 jingleConnectionManager
111 .getXmppConnectionService()
112 .getApplicationContext()));
113 this.callIntegration.setAddress(
114 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
115 this.callIntegration.setInitialized();
116 }
117
118 JingleRtpConnection(
119 final JingleConnectionManager jingleConnectionManager,
120 final Id id,
121 final Jid initiator,
122 final CallIntegration callIntegration) {
123 super(jingleConnectionManager, id, initiator);
124 final Conversation conversation =
125 jingleConnectionManager
126 .getXmppConnectionService()
127 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
128 this.message =
129 new Message(
130 conversation,
131 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
132 Message.TYPE_RTP_SESSION,
133 id.sessionId);
134 this.callIntegration = callIntegration;
135 this.callIntegration.setCallback(this);
136 }
137
138 @Override
139 synchronized void deliverPacket(final JinglePacket jinglePacket) {
140 switch (jinglePacket.getAction()) {
141 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
142 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
143 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
144 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
145 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
146 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
147 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
148 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
149 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
150 default -> {
151 respondOk(jinglePacket);
152 Log.d(
153 Config.LOGTAG,
154 String.format(
155 "%s: received unhandled jingle action %s",
156 id.account.getJid().asBareJid(), jinglePacket.getAction()));
157 }
158 }
159 }
160
161 @Override
162 synchronized void notifyRebound() {
163 if (isTerminated()) {
164 return;
165 }
166 webRTCWrapper.close();
167 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
168 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
169 }
170 if (isInState(
171 State.SESSION_INITIALIZED,
172 State.SESSION_INITIALIZED_PRE_APPROVED,
173 State.SESSION_ACCEPTED)) {
174 // we might have already changed resources (full jid) at this point; so this might not
175 // even reach the other party
176 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
177 } else {
178 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
179 finish();
180 }
181 }
182
183 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
184 respondOk(jinglePacket);
185 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
186 final State previous = this.state;
187 Log.d(
188 Config.LOGTAG,
189 id.account.getJid().asBareJid()
190 + ": received session terminate reason="
191 + wrapper.reason
192 + "("
193 + Strings.nullToEmpty(wrapper.text)
194 + ") while in state "
195 + previous);
196 if (TERMINATED.contains(previous)) {
197 Log.d(
198 Config.LOGTAG,
199 id.account.getJid().asBareJid()
200 + ": ignoring session terminate because already in "
201 + previous);
202 return;
203 }
204 webRTCWrapper.close();
205 final State target = reasonToState(wrapper.reason);
206 transitionOrThrow(target);
207 writeLogMessage(target);
208 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
209 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
210 }
211 finish();
212 }
213
214 private void receiveTransportInfo(final JinglePacket jinglePacket) {
215 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
216 // INITIALIZED only after transport-info has been received
217 if (isInState(
218 State.NULL,
219 State.PROCEED,
220 State.SESSION_INITIALIZED,
221 State.SESSION_INITIALIZED_PRE_APPROVED,
222 State.SESSION_ACCEPTED)) {
223 final RtpContentMap contentMap;
224 try {
225 contentMap = RtpContentMap.of(jinglePacket);
226 } catch (final IllegalArgumentException | NullPointerException e) {
227 Log.d(
228 Config.LOGTAG,
229 id.account.getJid().asBareJid()
230 + ": improperly formatted contents; ignoring",
231 e);
232 respondOk(jinglePacket);
233 return;
234 }
235 receiveTransportInfo(jinglePacket, contentMap);
236 } else {
237 if (isTerminated()) {
238 respondOk(jinglePacket);
239 Log.d(
240 Config.LOGTAG,
241 id.account.getJid().asBareJid()
242 + ": ignoring out-of-order transport info; we where already terminated");
243 } else {
244 Log.d(
245 Config.LOGTAG,
246 id.account.getJid().asBareJid()
247 + ": received transport info while in state="
248 + this.state);
249 terminateWithOutOfOrder(jinglePacket);
250 }
251 }
252 }
253
254 private void receiveTransportInfo(
255 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
256 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
257 candidates = contentMap.contents.entrySet();
258 final RtpContentMap remote = getRemoteContentMap();
259 final Set<String> remoteContentIds =
260 remote == null ? Collections.emptySet() : remote.contents.keySet();
261 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
262 Log.d(
263 Config.LOGTAG,
264 "received transport-info for unknown contents "
265 + contentMap.contents.keySet()
266 + " (known: "
267 + remoteContentIds
268 + ")");
269 respondOk(jinglePacket);
270 pendingIceCandidates.addAll(candidates);
271 return;
272 }
273 if (this.state != State.SESSION_ACCEPTED) {
274 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
275 respondOk(jinglePacket);
276 pendingIceCandidates.addAll(candidates);
277 return;
278 }
279 // zero candidates + modified credentials are an ICE restart offer
280 if (checkForIceRestart(jinglePacket, contentMap)) {
281 return;
282 }
283 respondOk(jinglePacket);
284 try {
285 processCandidates(candidates);
286 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
287 Log.w(
288 Config.LOGTAG,
289 id.account.getJid().asBareJid()
290 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
291 }
292 }
293
294 private void receiveContentAdd(final JinglePacket jinglePacket) {
295 final RtpContentMap modification;
296 try {
297 modification = RtpContentMap.of(jinglePacket);
298 modification.requireContentDescriptions();
299 } catch (final RuntimeException e) {
300 Log.d(
301 Config.LOGTAG,
302 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
303 Throwables.getRootCause(e));
304 respondOk(jinglePacket);
305 webRTCWrapper.close();
306 sendSessionTerminate(Reason.of(e), e.getMessage());
307 return;
308 }
309 if (isInState(State.SESSION_ACCEPTED)) {
310 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
311 final ListenableFuture<RtpContentMap> future =
312 receiveRtpContentMap(
313 modification,
314 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
315 Futures.addCallback(
316 future,
317 new FutureCallback<>() {
318 @Override
319 public void onSuccess(final RtpContentMap rtpContentMap) {
320 receiveContentAdd(jinglePacket, rtpContentMap);
321 }
322
323 @Override
324 public void onFailure(@NonNull Throwable throwable) {
325 respondOk(jinglePacket);
326 final Throwable rootCause = Throwables.getRootCause(throwable);
327 Log.d(
328 Config.LOGTAG,
329 id.account.getJid().asBareJid()
330 + ": improperly formatted contents in content-add",
331 throwable);
332 webRTCWrapper.close();
333 sendSessionTerminate(
334 Reason.ofThrowable(rootCause), rootCause.getMessage());
335 }
336 },
337 MoreExecutors.directExecutor());
338 } else {
339 terminateWithOutOfOrder(jinglePacket);
340 }
341 }
342
343 private void receiveContentAdd(
344 final JinglePacket jinglePacket, final RtpContentMap modification) {
345 final RtpContentMap remote = getRemoteContentMap();
346 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
347 respondOk(jinglePacket);
348 this.webRTCWrapper.close();
349 sendSessionTerminate(
350 Reason.FAILED_APPLICATION,
351 String.format(
352 "contents with names %s already exists",
353 Joiner.on(", ").join(modification.getNames())));
354 return;
355 }
356 final ContentAddition contentAddition =
357 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
358
359 final RtpContentMap outgoing = this.outgoingContentAdd;
360 final Set<ContentAddition.Summary> outgoingContentAddSummary =
361 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
362
363 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
364 if (isInitiator()) {
365 Log.d(
366 Config.LOGTAG,
367 id.getAccount().getJid().asBareJid()
368 + ": respond with tie break to matching content-add offer");
369 respondWithTieBreak(jinglePacket);
370 } else {
371 Log.d(
372 Config.LOGTAG,
373 id.getAccount().getJid().asBareJid()
374 + ": automatically accept matching content-add offer");
375 acceptContentAdd(contentAddition.summary, modification);
376 }
377 return;
378 }
379
380 // once we can display multiple video tracks we can be more loose with this condition
381 // theoretically it should also be fine to automatically accept audio only contents
382 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
383 Log.d(
384 Config.LOGTAG,
385 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
386 this.incomingContentAdd = modification;
387 respondOk(jinglePacket);
388 updateEndUserState();
389 } else {
390 respondOk(jinglePacket);
391 // TODO do we want to add a reason?
392 rejectContentAdd(modification);
393 }
394 }
395
396 private void receiveContentAccept(final JinglePacket jinglePacket) {
397 final RtpContentMap receivedContentAccept;
398 try {
399 receivedContentAccept = RtpContentMap.of(jinglePacket);
400 receivedContentAccept.requireContentDescriptions();
401 } catch (final RuntimeException e) {
402 Log.d(
403 Config.LOGTAG,
404 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
405 Throwables.getRootCause(e));
406 respondOk(jinglePacket);
407 webRTCWrapper.close();
408 sendSessionTerminate(Reason.of(e), e.getMessage());
409 return;
410 }
411
412 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
413 if (outgoingContentAdd == null) {
414 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
415 terminateWithOutOfOrder(jinglePacket);
416 return;
417 }
418 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
419 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
420 this.outgoingContentAdd = null;
421 respondOk(jinglePacket);
422 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
423 final ListenableFuture<RtpContentMap> future =
424 receiveRtpContentMap(
425 receivedContentAccept,
426 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
427 Futures.addCallback(
428 future,
429 new FutureCallback<>() {
430 @Override
431 public void onSuccess(final RtpContentMap result) {
432 receiveContentAccept(result);
433 }
434
435 @Override
436 public void onFailure(@NonNull final Throwable throwable) {
437 webRTCWrapper.close();
438 sendSessionTerminate(
439 Reason.ofThrowable(throwable), throwable.getMessage());
440 }
441 },
442 MoreExecutors.directExecutor());
443 } else {
444 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
445 terminateWithOutOfOrder(jinglePacket);
446 }
447 }
448
449 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
450 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
451 final RtpContentMap modifiedContentMap =
452 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
453
454 setRemoteContentMap(modifiedContentMap);
455
456 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
457
458 final org.webrtc.SessionDescription sdp =
459 new org.webrtc.SessionDescription(
460 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
461
462 try {
463 this.webRTCWrapper.setRemoteDescription(sdp).get();
464 } catch (final Exception e) {
465 final Throwable cause = Throwables.getRootCause(e);
466 Log.d(
467 Config.LOGTAG,
468 id.getAccount().getJid().asBareJid()
469 + ": unable to set remote description after receiving content-accept",
470 cause);
471 webRTCWrapper.close();
472 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
473 return;
474 }
475 Log.d(
476 Config.LOGTAG,
477 id.getAccount().getJid().asBareJid()
478 + ": remote has accepted content-add "
479 + ContentAddition.summary(receivedContentAccept));
480 processCandidates(receivedContentAccept.contents.entrySet());
481 updateEndUserState();
482 }
483
484 private void receiveContentModify(final JinglePacket jinglePacket) {
485 if (this.state != State.SESSION_ACCEPTED) {
486 terminateWithOutOfOrder(jinglePacket);
487 return;
488 }
489 final Map<String, Content.Senders> modification =
490 Maps.transformEntries(
491 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
492 final boolean isInitiator = isInitiator();
493 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
494 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
495 final Set<String> currentOutgoingMediaIds =
496 currentOutgoing == null
497 ? Collections.emptySet()
498 : currentOutgoing.contents.keySet();
499 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
500 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
501 respondOk(jinglePacket);
502 final RtpContentMap modifiedContentMap;
503 try {
504 modifiedContentMap =
505 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
506 } catch (final IllegalArgumentException e) {
507 webRTCWrapper.close();
508 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
509 return;
510 }
511 this.outgoingContentAdd = modifiedContentMap;
512 Log.d(
513 Config.LOGTAG,
514 id.account.getJid().asBareJid()
515 + ": processed content-modification for pending content-add");
516 } else if (remoteContentMap != null
517 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
518 respondOk(jinglePacket);
519 final RtpContentMap modifiedRemoteContentMap;
520 try {
521 modifiedRemoteContentMap =
522 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
523 } catch (final IllegalArgumentException e) {
524 webRTCWrapper.close();
525 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
526 return;
527 }
528 final SessionDescription offer;
529 try {
530 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
531 } catch (final IllegalArgumentException | NullPointerException e) {
532 Log.d(
533 Config.LOGTAG,
534 id.getAccount().getJid().asBareJid()
535 + ": unable convert offer from content-modify to SDP",
536 e);
537 webRTCWrapper.close();
538 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
539 return;
540 }
541 Log.d(
542 Config.LOGTAG,
543 id.account.getJid().asBareJid() + ": auto accepting content-modification");
544 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
545 } else {
546 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
547 respondWithItemNotFound(jinglePacket);
548 }
549 }
550
551 private void autoAcceptContentModify(
552 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
553 this.setRemoteContentMap(modifiedRemoteContentMap);
554 final org.webrtc.SessionDescription sdp =
555 new org.webrtc.SessionDescription(
556 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
557 try {
558 this.webRTCWrapper.setRemoteDescription(sdp).get();
559 // auto accept is only done when we already have tracks
560 final SessionDescription answer = setLocalSessionDescription();
561 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
562 modifyLocalContentMap(rtpContentMap);
563 // we do not need to send an answer but do we have to resend the candidates currently in
564 // SDP?
565 // resendCandidatesFromSdp(answer);
566 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
567 } catch (final Exception e) {
568 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
569 webRTCWrapper.close();
570 sendSessionTerminate(Reason.FAILED_APPLICATION);
571 }
572 }
573
574 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
575 final SessionDescription answer) {
576 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
577 new ImmutableMultimap.Builder<>();
578 for (final SessionDescription.Media media : answer.media) {
579 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
580 if (Strings.isNullOrEmpty(mid)) {
581 continue;
582 }
583 for (final String sdpCandidate : media.attributes.get("candidate")) {
584 final IceUdpTransportInfo.Candidate candidate =
585 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
586 if (candidate != null) {
587 candidateBuilder.put(mid, candidate);
588 }
589 }
590 }
591 return candidateBuilder.build();
592 }
593
594 private void receiveContentReject(final JinglePacket jinglePacket) {
595 final RtpContentMap receivedContentReject;
596 try {
597 receivedContentReject = RtpContentMap.of(jinglePacket);
598 } catch (final RuntimeException e) {
599 Log.d(
600 Config.LOGTAG,
601 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
602 Throwables.getRootCause(e));
603 respondOk(jinglePacket);
604 this.webRTCWrapper.close();
605 sendSessionTerminate(Reason.of(e), e.getMessage());
606 return;
607 }
608
609 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
610 if (outgoingContentAdd == null) {
611 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
612 terminateWithOutOfOrder(jinglePacket);
613 return;
614 }
615 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
616 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
617 this.outgoingContentAdd = null;
618 respondOk(jinglePacket);
619 Log.d(Config.LOGTAG, jinglePacket.toString());
620 receiveContentReject(ourSummary);
621 } else {
622 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
623 terminateWithOutOfOrder(jinglePacket);
624 }
625 }
626
627 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
628 try {
629 this.webRTCWrapper.removeTrack(Media.VIDEO);
630 final RtpContentMap localContentMap = customRollback();
631 modifyLocalContentMap(localContentMap);
632 } catch (final Exception e) {
633 final Throwable cause = Throwables.getRootCause(e);
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid().asBareJid()
637 + ": unable to rollback local description after receiving content-reject",
638 cause);
639 webRTCWrapper.close();
640 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
641 return;
642 }
643 Log.d(
644 Config.LOGTAG,
645 id.getAccount().getJid().asBareJid()
646 + ": remote has rejected our content-add "
647 + summary);
648 }
649
650 private void receiveContentRemove(final JinglePacket jinglePacket) {
651 final RtpContentMap receivedContentRemove;
652 try {
653 receivedContentRemove = RtpContentMap.of(jinglePacket);
654 receivedContentRemove.requireContentDescriptions();
655 } catch (final RuntimeException e) {
656 Log.d(
657 Config.LOGTAG,
658 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
659 Throwables.getRootCause(e));
660 respondOk(jinglePacket);
661 this.webRTCWrapper.close();
662 sendSessionTerminate(Reason.of(e), e.getMessage());
663 return;
664 }
665 respondOk(jinglePacket);
666 receiveContentRemove(receivedContentRemove);
667 }
668
669 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
670 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
671 final Set<ContentAddition.Summary> contentAddSummary =
672 incomingContentAdd == null
673 ? Collections.emptySet()
674 : ContentAddition.summary(incomingContentAdd);
675 final Set<ContentAddition.Summary> removeSummary =
676 ContentAddition.summary(receivedContentRemove);
677 if (contentAddSummary.equals(removeSummary)) {
678 this.incomingContentAdd = null;
679 updateEndUserState();
680 } else {
681 webRTCWrapper.close();
682 sendSessionTerminate(
683 Reason.FAILED_APPLICATION,
684 String.format(
685 "%s only supports %s as a means to retract a not yet accepted %s",
686 BuildConfig.APP_NAME,
687 JinglePacket.Action.CONTENT_REMOVE,
688 JinglePacket.Action.CONTENT_ADD));
689 }
690 }
691
692 public synchronized void retractContentAdd() {
693 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
694 if (outgoingContentAdd == null) {
695 throw new IllegalStateException("Not outgoing content add");
696 }
697 try {
698 webRTCWrapper.removeTrack(Media.VIDEO);
699 final RtpContentMap localContentMap = customRollback();
700 modifyLocalContentMap(localContentMap);
701 } catch (final Exception e) {
702 final Throwable cause = Throwables.getRootCause(e);
703 Log.d(
704 Config.LOGTAG,
705 id.getAccount().getJid().asBareJid()
706 + ": unable to rollback local description after trying to retract content-add",
707 cause);
708 webRTCWrapper.close();
709 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
710 return;
711 }
712 this.outgoingContentAdd = null;
713 final JinglePacket retract =
714 outgoingContentAdd
715 .toStub()
716 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
717 this.send(retract);
718 Log.d(
719 Config.LOGTAG,
720 id.getAccount().getJid()
721 + ": retract content-add "
722 + ContentAddition.summary(outgoingContentAdd));
723 }
724
725 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
726 final SessionDescription sdp = setLocalSessionDescription();
727 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
728 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
729 this.webRTCWrapper
730 .setRemoteDescription(
731 new org.webrtc.SessionDescription(
732 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
733 .get();
734 return localRtpContentMap;
735 }
736
737 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
738 final RtpContentMap currentRemote = getRemoteContentMap();
739 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
740 if (diff.isEmpty()) {
741 throw new IllegalStateException(
742 "Unexpected rollback condition. No difference between local and remote");
743 }
744 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
745 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
746 final RtpContentMap nextRemote =
747 currentRemote.addContent(
748 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
749 return SessionDescription.of(nextRemote, isResponder());
750 }
751 throw new IllegalStateException(
752 "Unexpected rollback condition. Senders were not uniformly none");
753 }
754
755 public synchronized void acceptContentAdd(
756 @NonNull final Set<ContentAddition.Summary> contentAddition) {
757 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
758 if (incomingContentAdd == null) {
759 throw new IllegalStateException("No incoming content add");
760 }
761
762 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
763 this.incomingContentAdd = null;
764 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
765 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
766 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
767 Log.d(
768 Config.LOGTAG,
769 "content addition is receive only. we want to upgrade to 'both'");
770 final RtpContentMap modifiedSenders =
771 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
772 final JinglePacket proposedContentModification =
773 modifiedSenders
774 .toStub()
775 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
776 proposedContentModification.setTo(id.with);
777 xmppConnectionService.sendIqPacket(
778 id.account,
779 proposedContentModification,
780 (account, response) -> {
781 if (response.getType() == IqPacket.TYPE.RESULT) {
782 Log.d(
783 Config.LOGTAG,
784 id.account.getJid().asBareJid()
785 + ": remote has accepted our upgrade to senders=both");
786 acceptContentAdd(
787 ContentAddition.summary(modifiedSenders), modifiedSenders);
788 } else {
789 Log.d(
790 Config.LOGTAG,
791 id.account.getJid().asBareJid()
792 + ": remote has rejected our upgrade to senders=both");
793 acceptContentAdd(contentAddition, incomingContentAdd);
794 }
795 });
796 } else {
797 acceptContentAdd(contentAddition, incomingContentAdd);
798 }
799 } else {
800 throw new IllegalStateException(
801 "Accepted content add does not match pending content-add");
802 }
803 }
804
805 private void acceptContentAdd(
806 @NonNull final Set<ContentAddition.Summary> contentAddition,
807 final RtpContentMap incomingContentAdd) {
808 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
809 final RtpContentMap modifiedContentMap =
810 getRemoteContentMap().addContent(incomingContentAdd, setup);
811 this.setRemoteContentMap(modifiedContentMap);
812
813 final SessionDescription offer;
814 try {
815 offer = SessionDescription.of(modifiedContentMap, isResponder());
816 } catch (final IllegalArgumentException | NullPointerException e) {
817 Log.d(
818 Config.LOGTAG,
819 id.getAccount().getJid().asBareJid()
820 + ": unable convert offer from content-add to SDP",
821 e);
822 webRTCWrapper.close();
823 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
824 return;
825 }
826 this.incomingContentAdd = null;
827 acceptContentAdd(contentAddition, offer);
828 }
829
830 private void acceptContentAdd(
831 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
832 final org.webrtc.SessionDescription sdp =
833 new org.webrtc.SessionDescription(
834 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
835 try {
836 this.webRTCWrapper.setRemoteDescription(sdp).get();
837
838 // TODO add tracks for 'media' where contentAddition.senders matches
839
840 // TODO if senders.sending(isInitiator())
841
842 this.webRTCWrapper.addTrack(Media.VIDEO);
843
844 // TODO add additional transceivers for recv only cases
845
846 final SessionDescription answer = setLocalSessionDescription();
847 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
848
849 final RtpContentMap contentAcceptMap =
850 rtpContentMap.toContentModification(
851 Collections2.transform(contentAddition, ca -> ca.name));
852
853 Log.d(
854 Config.LOGTAG,
855 id.getAccount().getJid().asBareJid()
856 + ": sending content-accept "
857 + ContentAddition.summary(contentAcceptMap));
858
859 addIceCandidatesFromBlackLog();
860
861 modifyLocalContentMap(rtpContentMap);
862 final ListenableFuture<RtpContentMap> future =
863 prepareOutgoingContentMap(contentAcceptMap);
864 Futures.addCallback(
865 future,
866 new FutureCallback<>() {
867 @Override
868 public void onSuccess(final RtpContentMap rtpContentMap) {
869 sendContentAccept(rtpContentMap);
870 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
871 }
872
873 @Override
874 public void onFailure(@NonNull final Throwable throwable) {
875 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
876 }
877 },
878 MoreExecutors.directExecutor());
879 } catch (final Exception e) {
880 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
881 webRTCWrapper.close();
882 sendSessionTerminate(Reason.FAILED_APPLICATION);
883 }
884 }
885
886 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
887 final JinglePacket jinglePacket =
888 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
889 send(jinglePacket);
890 }
891
892 public synchronized void rejectContentAdd() {
893 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
894 if (incomingContentAdd == null) {
895 throw new IllegalStateException("No incoming content add");
896 }
897 this.incomingContentAdd = null;
898 updateEndUserState();
899 rejectContentAdd(incomingContentAdd);
900 }
901
902 private void rejectContentAdd(final RtpContentMap contentMap) {
903 final JinglePacket jinglePacket =
904 contentMap
905 .toStub()
906 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
907 Log.d(
908 Config.LOGTAG,
909 id.getAccount().getJid().asBareJid()
910 + ": rejecting content "
911 + ContentAddition.summary(contentMap));
912 send(jinglePacket);
913 }
914
915 private boolean checkForIceRestart(
916 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
917 final RtpContentMap existing = getRemoteContentMap();
918 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
919 final IceUdpTransportInfo.Credentials newCredentials;
920 try {
921 existingCredentials = existing.getCredentials();
922 newCredentials = rtpContentMap.getDistinctCredentials();
923 } catch (final IllegalStateException e) {
924 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
925 return false;
926 }
927 if (existingCredentials.contains(newCredentials)) {
928 return false;
929 }
930 // TODO an alternative approach is to check if we already got an iq result to our
931 // ICE-restart
932 // and if that's the case we are seeing an answer.
933 // This might be more spec compliant but also more error prone potentially
934 final boolean isSignalStateStable =
935 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
936 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
937 final boolean isOffer = rtpContentMap.emptyCandidates();
938 final RtpContentMap restartContentMap;
939 try {
940 if (isOffer) {
941 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
942 restartContentMap =
943 existing.modifiedCredentials(
944 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
945 } else {
946 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
947 Log.d(
948 Config.LOGTAG,
949 "received confirmation of ICE restart"
950 + newCredentials
951 + " peer_setup="
952 + setup);
953 // DTLS setup attribute needs to be rewritten to reflect current peer state
954 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
955 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
956 }
957 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
958 return isOffer;
959 } else {
960 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
961 respondWithTieBreak(jinglePacket);
962 return true;
963 }
964 } catch (final Exception exception) {
965 respondOk(jinglePacket);
966 final Throwable rootCause = Throwables.getRootCause(exception);
967 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
968 // If this happens a termination is already in progress
969 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
970 return true;
971 }
972 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
973 webRTCWrapper.close();
974 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
975 return true;
976 }
977 }
978
979 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
980 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
981 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
982 throw new IllegalStateException("Invalid peer setup");
983 }
984 return peerSetup;
985 }
986
987 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
988 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
989 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
990 }
991 this.peerDtlsSetup = setup;
992 }
993
994 private boolean applyIceRestart(
995 final JinglePacket jinglePacket,
996 final RtpContentMap restartContentMap,
997 final boolean isOffer)
998 throws ExecutionException, InterruptedException {
999 final SessionDescription sessionDescription =
1000 SessionDescription.of(restartContentMap, isResponder());
1001 final org.webrtc.SessionDescription.Type type =
1002 isOffer
1003 ? org.webrtc.SessionDescription.Type.OFFER
1004 : org.webrtc.SessionDescription.Type.ANSWER;
1005 org.webrtc.SessionDescription sdp =
1006 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1007 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1008 if (isInitiator()) {
1009 // We ignore the offer and respond with tie-break. This will clause the responder
1010 // not to apply the content map
1011 return false;
1012 }
1013 }
1014 webRTCWrapper.setRemoteDescription(sdp).get();
1015 setRemoteContentMap(restartContentMap);
1016 if (isOffer) {
1017 final SessionDescription localSessionDescription = setLocalSessionDescription();
1018 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1019 // We need to respond OK before sending any candidates
1020 respondOk(jinglePacket);
1021 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1022 } else {
1023 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1024 }
1025 return true;
1026 }
1027
1028 private void processCandidates(
1029 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1030 contents) {
1031 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1032 content : contents) {
1033 processCandidate(content);
1034 }
1035 }
1036
1037 private void processCandidate(
1038 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1039 content) {
1040 final RtpContentMap rtpContentMap = getRemoteContentMap();
1041 final List<String> indices = toIdentificationTags(rtpContentMap);
1042 final String sdpMid = content.getKey(); // aka content name
1043 final IceUdpTransportInfo transport = content.getValue().transport;
1044 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1045
1046 // TODO check that credentials remained the same
1047
1048 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1049 final String sdp;
1050 try {
1051 sdp = candidate.toSdpAttribute(credentials.ufrag);
1052 } catch (final IllegalArgumentException e) {
1053 Log.d(
1054 Config.LOGTAG,
1055 id.account.getJid().asBareJid()
1056 + ": ignoring invalid ICE candidate "
1057 + e.getMessage());
1058 continue;
1059 }
1060 final int mLineIndex = indices.indexOf(sdpMid);
1061 if (mLineIndex < 0) {
1062 Log.w(
1063 Config.LOGTAG,
1064 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1065 }
1066 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1067 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1068 this.webRTCWrapper.addIceCandidate(iceCandidate);
1069 }
1070 }
1071
1072 private RtpContentMap getRemoteContentMap() {
1073 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1074 }
1075
1076 private RtpContentMap getLocalContentMap() {
1077 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1078 }
1079
1080 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1081 final Group originalGroup = rtpContentMap.group;
1082 final List<String> identificationTags =
1083 originalGroup == null
1084 ? rtpContentMap.getNames()
1085 : originalGroup.getIdentificationTags();
1086 if (identificationTags.size() == 0) {
1087 Log.w(
1088 Config.LOGTAG,
1089 id.account.getJid().asBareJid()
1090 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1091 }
1092 return identificationTags;
1093 }
1094
1095 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1096 final JinglePacket jinglePacket, final boolean expectVerification) {
1097 try {
1098 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1099 } catch (final Exception e) {
1100 return Futures.immediateFailedFuture(e);
1101 }
1102 }
1103
1104 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1105 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1106 Log.d(
1107 Config.LOGTAG,
1108 "receiveRtpContentMap("
1109 + receivedContentMap.getClass().getSimpleName()
1110 + ",expectVerification="
1111 + expectVerification
1112 + ")");
1113 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1114 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1115 id.account
1116 .getAxolotlService()
1117 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1118 return Futures.transform(
1119 future,
1120 omemoVerifiedPayload -> {
1121 // TODO test if an exception here triggers a correct abort
1122 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1123 Log.d(
1124 Config.LOGTAG,
1125 id.account.getJid().asBareJid()
1126 + ": received verifiable DTLS fingerprint via "
1127 + omemoVerification);
1128 return omemoVerifiedPayload.getPayload();
1129 },
1130 MoreExecutors.directExecutor());
1131 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1132 return Futures.immediateFailedFuture(
1133 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1134 } else {
1135 return Futures.immediateFuture(receivedContentMap);
1136 }
1137 }
1138
1139 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1140 if (isInitiator()) {
1141 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1142 return;
1143 }
1144 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1145 Futures.addCallback(
1146 future,
1147 new FutureCallback<>() {
1148 @Override
1149 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1150 receiveSessionInitiate(jinglePacket, rtpContentMap);
1151 }
1152
1153 @Override
1154 public void onFailure(@NonNull final Throwable throwable) {
1155 respondOk(jinglePacket);
1156 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1157 }
1158 },
1159 MoreExecutors.directExecutor());
1160 }
1161
1162 private void receiveSessionInitiate(
1163 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1164 try {
1165 contentMap.requireContentDescriptions();
1166 contentMap.requireDTLSFingerprint(true);
1167 } catch (final RuntimeException e) {
1168 Log.d(
1169 Config.LOGTAG,
1170 id.account.getJid().asBareJid() + ": improperly formatted contents",
1171 Throwables.getRootCause(e));
1172 respondOk(jinglePacket);
1173 sendSessionTerminate(Reason.of(e), e.getMessage());
1174 return;
1175 }
1176 Log.d(
1177 Config.LOGTAG,
1178 "processing session-init with " + contentMap.contents.size() + " contents");
1179 final State target;
1180 if (this.state == State.PROCEED) {
1181 Preconditions.checkState(
1182 proposedMedia != null && proposedMedia.size() > 0,
1183 "proposed media must be set when processing pre-approved session-initiate");
1184 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1185 sendSessionTerminate(
1186 Reason.SECURITY_ERROR,
1187 String.format(
1188 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1189 this.proposedMedia, contentMap.getMedia()));
1190 return;
1191 }
1192 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1193 } else {
1194 target = State.SESSION_INITIALIZED;
1195 setProposedMedia(contentMap.getMedia());
1196 }
1197 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1198 respondOk(jinglePacket);
1199 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1200 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1201 Log.d(
1202 Config.LOGTAG,
1203 id.account.getJid().asBareJid()
1204 + ": automatically accepting session-initiate");
1205 sendSessionAccept();
1206 } else {
1207 Log.d(
1208 Config.LOGTAG,
1209 id.account.getJid().asBareJid()
1210 + ": received not pre-approved session-initiate. start ringing");
1211 startRinging();
1212 }
1213 } else {
1214 Log.d(
1215 Config.LOGTAG,
1216 String.format(
1217 "%s: received session-initiate while in state %s",
1218 id.account.getJid().asBareJid(), state));
1219 terminateWithOutOfOrder(jinglePacket);
1220 }
1221 }
1222
1223 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1224 if (isResponder()) {
1225 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1226 return;
1227 }
1228 final ListenableFuture<RtpContentMap> future =
1229 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1230 Futures.addCallback(
1231 future,
1232 new FutureCallback<>() {
1233 @Override
1234 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1235 receiveSessionAccept(jinglePacket, rtpContentMap);
1236 }
1237
1238 @Override
1239 public void onFailure(@NonNull final Throwable throwable) {
1240 respondOk(jinglePacket);
1241 Log.d(
1242 Config.LOGTAG,
1243 id.account.getJid().asBareJid()
1244 + ": improperly formatted contents in session-accept",
1245 throwable);
1246 webRTCWrapper.close();
1247 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1248 }
1249 },
1250 MoreExecutors.directExecutor());
1251 }
1252
1253 private void receiveSessionAccept(
1254 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1255 try {
1256 contentMap.requireContentDescriptions();
1257 contentMap.requireDTLSFingerprint();
1258 } catch (final RuntimeException e) {
1259 respondOk(jinglePacket);
1260 Log.d(
1261 Config.LOGTAG,
1262 id.account.getJid().asBareJid()
1263 + ": improperly formatted contents in session-accept",
1264 e);
1265 webRTCWrapper.close();
1266 sendSessionTerminate(Reason.of(e), e.getMessage());
1267 return;
1268 }
1269 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1270 if (!initiatorMedia.equals(contentMap.getMedia())) {
1271 sendSessionTerminate(
1272 Reason.SECURITY_ERROR,
1273 String.format(
1274 "Your session-included included media %s but our session-initiate was %s",
1275 this.proposedMedia, contentMap.getMedia()));
1276 return;
1277 }
1278 Log.d(
1279 Config.LOGTAG,
1280 "processing session-accept with " + contentMap.contents.size() + " contents");
1281 if (transition(State.SESSION_ACCEPTED)) {
1282 respondOk(jinglePacket);
1283 receiveSessionAccept(contentMap);
1284 } else {
1285 Log.d(
1286 Config.LOGTAG,
1287 String.format(
1288 "%s: received session-accept while in state %s",
1289 id.account.getJid().asBareJid(), state));
1290 respondOk(jinglePacket);
1291 }
1292 }
1293
1294 private void receiveSessionAccept(final RtpContentMap contentMap) {
1295 this.responderRtpContentMap = contentMap;
1296 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1297 final SessionDescription sessionDescription;
1298 try {
1299 sessionDescription = SessionDescription.of(contentMap, false);
1300 } catch (final IllegalArgumentException | NullPointerException e) {
1301 Log.d(
1302 Config.LOGTAG,
1303 id.account.getJid().asBareJid()
1304 + ": unable convert offer from session-accept to SDP",
1305 e);
1306 webRTCWrapper.close();
1307 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1308 return;
1309 }
1310 final org.webrtc.SessionDescription answer =
1311 new org.webrtc.SessionDescription(
1312 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1313 try {
1314 this.webRTCWrapper.setRemoteDescription(answer).get();
1315 } catch (final Exception e) {
1316 Log.d(
1317 Config.LOGTAG,
1318 id.account.getJid().asBareJid()
1319 + ": unable to set remote description after receiving session-accept",
1320 Throwables.getRootCause(e));
1321 webRTCWrapper.close();
1322 sendSessionTerminate(
1323 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1324 return;
1325 }
1326 processCandidates(contentMap.contents.entrySet());
1327 }
1328
1329 private void sendSessionAccept() {
1330 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1331 if (rtpContentMap == null) {
1332 throw new IllegalStateException("initiator RTP Content Map has not been set");
1333 }
1334 final SessionDescription offer;
1335 try {
1336 offer = SessionDescription.of(rtpContentMap, true);
1337 } catch (final IllegalArgumentException | NullPointerException e) {
1338 Log.d(
1339 Config.LOGTAG,
1340 id.account.getJid().asBareJid()
1341 + ": unable convert offer from session-initiate to SDP",
1342 e);
1343 webRTCWrapper.close();
1344 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1345 return;
1346 }
1347 sendSessionAccept(rtpContentMap.getMedia(), offer);
1348 }
1349
1350 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1351 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1352 }
1353
1354 private synchronized void sendSessionAccept(
1355 final Set<Media> media,
1356 final SessionDescription offer,
1357 final List<PeerConnection.IceServer> iceServers) {
1358 if (isTerminated()) {
1359 Log.w(
1360 Config.LOGTAG,
1361 id.account.getJid().asBareJid()
1362 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1363 return;
1364 }
1365 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1366 try {
1367 setupWebRTC(media, iceServers, !includeCandidates);
1368 } catch (final WebRTCWrapper.InitializationException e) {
1369 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1370 webRTCWrapper.close();
1371 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1372 return;
1373 }
1374 final org.webrtc.SessionDescription sdp =
1375 new org.webrtc.SessionDescription(
1376 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1377 try {
1378 this.webRTCWrapper.setRemoteDescription(sdp).get();
1379 addIceCandidatesFromBlackLog();
1380 org.webrtc.SessionDescription webRTCSessionDescription =
1381 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1382 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1383 } catch (final Exception e) {
1384 failureToAcceptSession(e);
1385 }
1386 }
1387
1388 private void failureToAcceptSession(final Throwable throwable) {
1389 if (isTerminated()) {
1390 return;
1391 }
1392 final Throwable rootCause = Throwables.getRootCause(throwable);
1393 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1394 webRTCWrapper.close();
1395 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1396 }
1397
1398 private void failureToPerformAction(
1399 final JinglePacket.Action action, final Throwable throwable) {
1400 if (isTerminated()) {
1401 return;
1402 }
1403 final Throwable rootCause = Throwables.getRootCause(throwable);
1404 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1405 webRTCWrapper.close();
1406 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1407 }
1408
1409 private void addIceCandidatesFromBlackLog() {
1410 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1411 while ((foo = this.pendingIceCandidates.poll()) != null) {
1412 processCandidate(foo);
1413 Log.d(
1414 Config.LOGTAG,
1415 id.account.getJid().asBareJid() + ": added candidate from back log");
1416 }
1417 }
1418
1419 private void prepareSessionAccept(
1420 final org.webrtc.SessionDescription webRTCSessionDescription,
1421 final boolean includeCandidates) {
1422 final SessionDescription sessionDescription =
1423 SessionDescription.parse(webRTCSessionDescription.description);
1424 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1425 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1426 if (includeCandidates) {
1427 candidates = parseCandidates(sessionDescription);
1428 } else {
1429 candidates = ImmutableMultimap.of();
1430 }
1431 this.responderRtpContentMap = respondingRtpContentMap;
1432 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1433 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1434 prepareOutgoingContentMap(respondingRtpContentMap);
1435 Futures.addCallback(
1436 outgoingContentMapFuture,
1437 new FutureCallback<>() {
1438 @Override
1439 public void onSuccess(final RtpContentMap outgoingContentMap) {
1440 if (includeCandidates) {
1441 Log.d(
1442 Config.LOGTAG,
1443 "including "
1444 + candidates.size()
1445 + " candidates in session accept");
1446 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1447 } else {
1448 sendSessionAccept(outgoingContentMap);
1449 }
1450 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1451 }
1452
1453 @Override
1454 public void onFailure(@NonNull Throwable throwable) {
1455 failureToAcceptSession(throwable);
1456 }
1457 },
1458 MoreExecutors.directExecutor());
1459 }
1460
1461 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1462 if (isTerminated()) {
1463 Log.w(
1464 Config.LOGTAG,
1465 id.account.getJid().asBareJid()
1466 + ": preparing session accept was too slow. already terminated. nothing to do.");
1467 return;
1468 }
1469 transitionOrThrow(State.SESSION_ACCEPTED);
1470 final JinglePacket sessionAccept =
1471 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1472 send(sessionAccept);
1473 }
1474
1475 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1476 final RtpContentMap rtpContentMap) {
1477 if (this.omemoVerification.hasDeviceId()) {
1478 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1479 verifiedPayloadFuture =
1480 id.account
1481 .getAxolotlService()
1482 .encrypt(
1483 rtpContentMap,
1484 id.with,
1485 omemoVerification.getDeviceId());
1486 return Futures.transform(
1487 verifiedPayloadFuture,
1488 verifiedPayload -> {
1489 omemoVerification.setOrEnsureEqual(verifiedPayload);
1490 return verifiedPayload.getPayload();
1491 },
1492 MoreExecutors.directExecutor());
1493 } else {
1494 return Futures.immediateFuture(rtpContentMap);
1495 }
1496 }
1497
1498 synchronized void deliveryMessage(
1499 final Jid from,
1500 final Element message,
1501 final String serverMessageId,
1502 final long timestamp) {
1503 Log.d(
1504 Config.LOGTAG,
1505 id.account.getJid().asBareJid()
1506 + ": delivered message to JingleRtpConnection "
1507 + message);
1508 switch (message.getName()) {
1509 case "propose" -> receivePropose(
1510 from, Propose.upgrade(message), serverMessageId, timestamp);
1511 case "proceed" -> receiveProceed(
1512 from, Proceed.upgrade(message), serverMessageId, timestamp);
1513 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1514 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1515 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1516 }
1517 }
1518
1519 void deliverFailedProceed(final String message) {
1520 Log.d(
1521 Config.LOGTAG,
1522 id.account.getJid().asBareJid()
1523 + ": receive message error for proceed message ("
1524 + Strings.nullToEmpty(message)
1525 + ")");
1526 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1527 webRTCWrapper.close();
1528 Log.d(
1529 Config.LOGTAG,
1530 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1531 this.finish();
1532 }
1533 }
1534
1535 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1536 final boolean originatedFromMyself =
1537 from.asBareJid().equals(id.account.getJid().asBareJid());
1538 if (originatedFromMyself) {
1539 if (transition(State.ACCEPTED)) {
1540 acceptedOnOtherDevice(serverMsgId, timestamp);
1541 } else {
1542 Log.d(
1543 Config.LOGTAG,
1544 id.account.getJid().asBareJid()
1545 + ": unable to transition to accept because already in state="
1546 + this.state);
1547 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1548 }
1549 } else {
1550 Log.d(
1551 Config.LOGTAG,
1552 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1553 }
1554 }
1555
1556 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1557 if (serverMsgId != null) {
1558 this.message.setServerMsgId(serverMsgId);
1559 }
1560 this.message.setTime(timestamp);
1561 this.message.setCarbon(true); // indicate that call was accepted on other device
1562 this.writeLogMessageSuccess(0);
1563 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1564 this.finish();
1565 }
1566
1567 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1568 final boolean originatedFromMyself =
1569 from.asBareJid().equals(id.account.getJid().asBareJid());
1570 // reject from another one of my clients
1571 if (originatedFromMyself) {
1572 receiveRejectFromMyself(serverMsgId, timestamp);
1573 } else if (isInitiator()) {
1574 if (from.equals(id.with)) {
1575 receiveRejectFromResponder();
1576 } else {
1577 Log.d(
1578 Config.LOGTAG,
1579 id.account.getJid()
1580 + ": ignoring reject from "
1581 + from
1582 + " for session with "
1583 + id.with);
1584 }
1585 } else {
1586 Log.d(
1587 Config.LOGTAG,
1588 id.account.getJid()
1589 + ": ignoring reject from "
1590 + from
1591 + " for session with "
1592 + id.with);
1593 }
1594 }
1595
1596 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1597 if (transition(State.REJECTED)) {
1598 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1599 this.finish();
1600 if (serverMsgId != null) {
1601 this.message.setServerMsgId(serverMsgId);
1602 }
1603 this.message.setTime(timestamp);
1604 this.message.setCarbon(true); // indicate that call was rejected on other device
1605 writeLogMessageMissed();
1606 } else {
1607 Log.d(
1608 Config.LOGTAG,
1609 "not able to transition into REJECTED because already in " + this.state);
1610 }
1611 }
1612
1613 private void receiveRejectFromResponder() {
1614 if (isInState(State.PROCEED)) {
1615 Log.d(
1616 Config.LOGTAG,
1617 id.account.getJid()
1618 + ": received reject while still in proceed. callee reconsidered");
1619 closeTransitionLogFinish(State.REJECTED_RACED);
1620 return;
1621 }
1622 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1623 Log.d(
1624 Config.LOGTAG,
1625 id.account.getJid()
1626 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1627 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1628 return;
1629 }
1630 Log.d(
1631 Config.LOGTAG,
1632 id.account.getJid()
1633 + ": ignoring reject from responder because already in state "
1634 + this.state);
1635 }
1636
1637 private void receivePropose(
1638 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1639 final boolean originatedFromMyself =
1640 from.asBareJid().equals(id.account.getJid().asBareJid());
1641 if (originatedFromMyself) {
1642 Log.d(
1643 Config.LOGTAG,
1644 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1645 } else if (transition(
1646 State.PROPOSED,
1647 () -> {
1648 final Collection<RtpDescription> descriptions =
1649 Collections2.transform(
1650 Collections2.filter(
1651 propose.getDescriptions(),
1652 d -> d instanceof RtpDescription),
1653 input -> (RtpDescription) input);
1654 final Collection<Media> media =
1655 Collections2.transform(descriptions, RtpDescription::getMedia);
1656 Preconditions.checkState(
1657 !media.contains(Media.UNKNOWN),
1658 "RTP descriptions contain unknown media");
1659 Log.d(
1660 Config.LOGTAG,
1661 id.account.getJid().asBareJid()
1662 + ": received session proposal from "
1663 + from
1664 + " for "
1665 + media);
1666 this.setProposedMedia(Sets.newHashSet(media));
1667 })) {
1668 if (serverMsgId != null) {
1669 this.message.setServerMsgId(serverMsgId);
1670 }
1671 this.message.setTime(timestamp);
1672 startRinging();
1673 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1674 sendJingleMessage("ringing");
1675 }
1676 } else {
1677 Log.d(
1678 Config.LOGTAG,
1679 id.account.getJid()
1680 + ": ignoring session proposal because already in "
1681 + state);
1682 }
1683 }
1684
1685 private void startRinging() {
1686 this.callIntegration.setRinging();
1687 Log.d(
1688 Config.LOGTAG,
1689 id.account.getJid().asBareJid()
1690 + ": received call from "
1691 + id.with
1692 + ". start ringing");
1693 ringingTimeoutFuture =
1694 jingleConnectionManager.schedule(
1695 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1696 if (CallIntegration.selfManaged(xmppConnectionService)) {
1697 return;
1698 }
1699 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1700 }
1701
1702 private synchronized void ringingTimeout() {
1703 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1704 switch (this.state) {
1705 case PROPOSED -> {
1706 message.markUnread();
1707 rejectCallFromProposed();
1708 }
1709 case SESSION_INITIALIZED -> {
1710 message.markUnread();
1711 rejectCallFromSessionInitiate();
1712 }
1713 }
1714 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1715 }
1716
1717 private void cancelRingingTimeout() {
1718 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1719 if (future != null && !future.isCancelled()) {
1720 future.cancel(false);
1721 }
1722 }
1723
1724 private void receiveProceed(
1725 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1726 final Set<Media> media =
1727 Preconditions.checkNotNull(
1728 this.proposedMedia, "Proposed media has to be set before handling proceed");
1729 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1730 if (from.equals(id.with)) {
1731 if (isInitiator()) {
1732 if (transition(State.PROCEED)) {
1733 if (serverMsgId != null) {
1734 this.message.setServerMsgId(serverMsgId);
1735 }
1736 this.message.setTime(timestamp);
1737 final Integer remoteDeviceId = proceed.getDeviceId();
1738 if (isOmemoEnabled()) {
1739 this.omemoVerification.setDeviceId(remoteDeviceId);
1740 } else {
1741 if (remoteDeviceId != null) {
1742 Log.d(
1743 Config.LOGTAG,
1744 id.account.getJid().asBareJid()
1745 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1746 }
1747 this.omemoVerification.setDeviceId(null);
1748 }
1749 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1750 } else {
1751 Log.d(
1752 Config.LOGTAG,
1753 String.format(
1754 "%s: ignoring proceed because already in %s",
1755 id.account.getJid().asBareJid(), this.state));
1756 }
1757 } else {
1758 Log.d(
1759 Config.LOGTAG,
1760 String.format(
1761 "%s: ignoring proceed because we were not initializing",
1762 id.account.getJid().asBareJid()));
1763 }
1764 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1765 if (transition(State.ACCEPTED)) {
1766 Log.d(
1767 Config.LOGTAG,
1768 id.account.getJid().asBareJid()
1769 + ": moved session with "
1770 + id.with
1771 + " into state accepted after received carbon copied proceed");
1772 acceptedOnOtherDevice(serverMsgId, timestamp);
1773 }
1774 } else {
1775 Log.d(
1776 Config.LOGTAG,
1777 String.format(
1778 "%s: ignoring proceed from %s. was expected from %s",
1779 id.account.getJid().asBareJid(), from, id.with));
1780 }
1781 }
1782
1783 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1784 if (from.equals(id.with)) {
1785 final State target =
1786 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1787 if (transition(target)) {
1788 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1789 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1790 Log.d(
1791 Config.LOGTAG,
1792 id.account.getJid().asBareJid()
1793 + ": session with "
1794 + id.with
1795 + " has been retracted (serverMsgId="
1796 + serverMsgId
1797 + ")");
1798 if (serverMsgId != null) {
1799 this.message.setServerMsgId(serverMsgId);
1800 }
1801 this.message.setTime(timestamp);
1802 if (target == State.RETRACTED) {
1803 this.message.markUnread();
1804 }
1805 writeLogMessageMissed();
1806 finish();
1807 } else {
1808 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1809 }
1810 } else {
1811 // TODO parse retract from self
1812 Log.d(
1813 Config.LOGTAG,
1814 id.account.getJid().asBareJid()
1815 + ": received retract from "
1816 + from
1817 + ". expected retract from"
1818 + id.with
1819 + ". ignoring");
1820 }
1821 }
1822
1823 public void sendSessionInitiate() {
1824 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1825 }
1826
1827 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1828 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1829 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1830 }
1831
1832 private synchronized void sendSessionInitiate(
1833 final Set<Media> media,
1834 final State targetState,
1835 final List<PeerConnection.IceServer> iceServers) {
1836 if (isTerminated()) {
1837 Log.w(
1838 Config.LOGTAG,
1839 id.account.getJid().asBareJid()
1840 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1841 return;
1842 }
1843 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1844 try {
1845 setupWebRTC(media, iceServers, !includeCandidates);
1846 } catch (final WebRTCWrapper.InitializationException e) {
1847 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1848 webRTCWrapper.close();
1849 sendRetract(Reason.ofThrowable(e));
1850 return;
1851 }
1852 try {
1853 org.webrtc.SessionDescription webRTCSessionDescription =
1854 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1855 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1856 } catch (final Exception e) {
1857 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1858 // exceptions
1859 failureToInitiateSession(e, targetState);
1860 }
1861 }
1862
1863 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1864 if (isTerminated()) {
1865 return;
1866 }
1867 Log.d(
1868 Config.LOGTAG,
1869 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1870 Throwables.getRootCause(throwable));
1871 webRTCWrapper.close();
1872 final Reason reason = Reason.ofThrowable(throwable);
1873 if (isInState(targetState)) {
1874 sendSessionTerminate(reason, throwable.getMessage());
1875 } else {
1876 sendRetract(reason);
1877 }
1878 }
1879
1880 private void sendRetract(final Reason reason) {
1881 // TODO embed reason into retract
1882 sendJingleMessage("retract", id.with.asBareJid());
1883 transitionOrThrow(reasonToState(reason));
1884 this.finish();
1885 }
1886
1887 private void prepareSessionInitiate(
1888 final org.webrtc.SessionDescription webRTCSessionDescription,
1889 final boolean includeCandidates,
1890 final State targetState) {
1891 final SessionDescription sessionDescription =
1892 SessionDescription.parse(webRTCSessionDescription.description);
1893 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1894 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1895 if (includeCandidates) {
1896 candidates = parseCandidates(sessionDescription);
1897 } else {
1898 candidates = ImmutableMultimap.of();
1899 }
1900 this.initiatorRtpContentMap = rtpContentMap;
1901 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1902 encryptSessionInitiate(rtpContentMap);
1903 Futures.addCallback(
1904 outgoingContentMapFuture,
1905 new FutureCallback<>() {
1906 @Override
1907 public void onSuccess(final RtpContentMap outgoingContentMap) {
1908 if (includeCandidates) {
1909 Log.d(
1910 Config.LOGTAG,
1911 "including "
1912 + candidates.size()
1913 + " candidates in session initiate");
1914 sendSessionInitiate(
1915 outgoingContentMap.withCandidates(candidates), targetState);
1916 } else {
1917 sendSessionInitiate(outgoingContentMap, targetState);
1918 }
1919 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1920 }
1921
1922 @Override
1923 public void onFailure(@NonNull final Throwable throwable) {
1924 failureToInitiateSession(throwable, targetState);
1925 }
1926 },
1927 MoreExecutors.directExecutor());
1928 }
1929
1930 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1931 if (isTerminated()) {
1932 Log.w(
1933 Config.LOGTAG,
1934 id.account.getJid().asBareJid()
1935 + ": preparing session was too slow. already terminated. nothing to do.");
1936 return;
1937 }
1938 this.transitionOrThrow(targetState);
1939 final JinglePacket sessionInitiate =
1940 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1941 send(sessionInitiate);
1942 }
1943
1944 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1945 final RtpContentMap rtpContentMap) {
1946 if (this.omemoVerification.hasDeviceId()) {
1947 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1948 verifiedPayloadFuture =
1949 id.account
1950 .getAxolotlService()
1951 .encrypt(
1952 rtpContentMap,
1953 id.with,
1954 omemoVerification.getDeviceId());
1955 final ListenableFuture<RtpContentMap> future =
1956 Futures.transform(
1957 verifiedPayloadFuture,
1958 verifiedPayload -> {
1959 omemoVerification.setSessionFingerprint(
1960 verifiedPayload.getFingerprint());
1961 return verifiedPayload.getPayload();
1962 },
1963 MoreExecutors.directExecutor());
1964 if (Config.REQUIRE_RTP_VERIFICATION) {
1965 return future;
1966 }
1967 return Futures.catching(
1968 future,
1969 CryptoFailedException.class,
1970 e -> {
1971 Log.w(
1972 Config.LOGTAG,
1973 id.account.getJid().asBareJid()
1974 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1975 e);
1976 return rtpContentMap;
1977 },
1978 MoreExecutors.directExecutor());
1979 } else {
1980 return Futures.immediateFuture(rtpContentMap);
1981 }
1982 }
1983
1984 protected void sendSessionTerminate(final Reason reason) {
1985 sendSessionTerminate(reason, null);
1986 }
1987
1988 protected void sendSessionTerminate(final Reason reason, final String text) {
1989 sendSessionTerminate(reason, text, this::writeLogMessage);
1990 sendJingleMessageFinish(reason);
1991 }
1992
1993 private void sendTransportInfo(
1994 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1995 final RtpContentMap transportInfo;
1996 try {
1997 final RtpContentMap rtpContentMap =
1998 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1999 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2000 } catch (final Exception e) {
2001 Log.d(
2002 Config.LOGTAG,
2003 id.account.getJid().asBareJid()
2004 + ": unable to prepare transport-info from candidate for content="
2005 + contentName);
2006 return;
2007 }
2008 final JinglePacket jinglePacket =
2009 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2010 send(jinglePacket);
2011 }
2012
2013 public RtpEndUserState getEndUserState() {
2014 switch (this.state) {
2015 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2016 if (isInitiator()) {
2017 return RtpEndUserState.RINGING;
2018 } else {
2019 return RtpEndUserState.INCOMING_CALL;
2020 }
2021 }
2022 case PROCEED -> {
2023 if (isInitiator()) {
2024 return RtpEndUserState.RINGING;
2025 } else {
2026 return RtpEndUserState.ACCEPTING_CALL;
2027 }
2028 }
2029 case SESSION_INITIALIZED_PRE_APPROVED -> {
2030 if (isInitiator()) {
2031 return RtpEndUserState.RINGING;
2032 } else {
2033 return RtpEndUserState.CONNECTING;
2034 }
2035 }
2036 case SESSION_ACCEPTED -> {
2037 final ContentAddition ca = getPendingContentAddition();
2038 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2039 return RtpEndUserState.INCOMING_CONTENT_ADD;
2040 }
2041 return getPeerConnectionStateAsEndUserState();
2042 }
2043 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2044 if (isInitiator()) {
2045 return RtpEndUserState.DECLINED_OR_BUSY;
2046 } else {
2047 return RtpEndUserState.ENDED;
2048 }
2049 }
2050 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2051 return RtpEndUserState.ENDED;
2052 }
2053 case RETRACTED_RACED -> {
2054 if (isInitiator()) {
2055 return RtpEndUserState.ENDED;
2056 } else {
2057 return RtpEndUserState.RETRACTED;
2058 }
2059 }
2060 case TERMINATED_CONNECTIVITY_ERROR -> {
2061 return zeroDuration()
2062 ? RtpEndUserState.CONNECTIVITY_ERROR
2063 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2064 }
2065 case TERMINATED_APPLICATION_FAILURE -> {
2066 return RtpEndUserState.APPLICATION_ERROR;
2067 }
2068 case TERMINATED_SECURITY_ERROR -> {
2069 return RtpEndUserState.SECURITY_ERROR;
2070 }
2071 }
2072 throw new IllegalStateException(
2073 String.format("%s has no equivalent EndUserState", this.state));
2074 }
2075
2076 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2077 final PeerConnection.PeerConnectionState state;
2078 try {
2079 state = webRTCWrapper.getState();
2080 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2081 // We usually close the WebRTCWrapper *before* transitioning so we might still
2082 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2083 return RtpEndUserState.ENDING_CALL;
2084 }
2085 return switch (state) {
2086 case CONNECTED -> RtpEndUserState.CONNECTED;
2087 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2088 case CLOSED -> RtpEndUserState.ENDING_CALL;
2089 default -> zeroDuration()
2090 ? RtpEndUserState.CONNECTIVITY_ERROR
2091 : RtpEndUserState.RECONNECTING;
2092 };
2093 }
2094
2095 private boolean isPeerConnectionConnected() {
2096 try {
2097 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2098 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2099 return false;
2100 }
2101 }
2102
2103 private void updateCallIntegrationState() {
2104 switch (this.state) {
2105 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2106 if (isInitiator()) {
2107 this.callIntegration.setDialing();
2108 } else {
2109 this.callIntegration.setRinging();
2110 }
2111 }
2112 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2113 if (isInitiator()) {
2114 this.callIntegration.setDialing();
2115 } else {
2116 this.callIntegration.setInitialized();
2117 }
2118 }
2119 case SESSION_ACCEPTED -> {
2120 if (isPeerConnectionConnected()) {
2121 this.callIntegration.setActive();
2122 } else {
2123 this.callIntegration.setInitialized();
2124 }
2125 }
2126 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2127 if (isInitiator()) {
2128 this.callIntegration.busy();
2129 } else {
2130 this.callIntegration.rejected();
2131 }
2132 }
2133 case TERMINATED_SUCCESS -> this.callIntegration.success();
2134 case ACCEPTED -> this.callIntegration.accepted();
2135 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2136 .retracted();
2137 case TERMINATED_CONNECTIVITY_ERROR,
2138 TERMINATED_APPLICATION_FAILURE,
2139 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2140 default -> throw new IllegalStateException(
2141 String.format("%s is not handled", this.state));
2142 }
2143 }
2144
2145 public ContentAddition getPendingContentAddition() {
2146 final RtpContentMap in = this.incomingContentAdd;
2147 final RtpContentMap out = this.outgoingContentAdd;
2148 if (out != null) {
2149 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2150 } else if (in != null) {
2151 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2152 } else {
2153 return null;
2154 }
2155 }
2156
2157 public Set<Media> getMedia() {
2158 final State current = getState();
2159 if (current == State.NULL) {
2160 if (isInitiator()) {
2161 return Preconditions.checkNotNull(
2162 this.proposedMedia, "RTP connection has not been initialized properly");
2163 }
2164 throw new IllegalStateException("RTP connection has not been initialized yet");
2165 }
2166 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2167 return Preconditions.checkNotNull(
2168 this.proposedMedia, "RTP connection has not been initialized properly");
2169 }
2170 final RtpContentMap localContentMap = getLocalContentMap();
2171 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2172 if (localContentMap != null) {
2173 return localContentMap.getMedia();
2174 } else if (initiatorContentMap != null) {
2175 return initiatorContentMap.getMedia();
2176 } else if (isTerminated()) {
2177 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2178 } else {
2179 return Preconditions.checkNotNull(
2180 this.proposedMedia, "RTP connection has not been initialized properly");
2181 }
2182 }
2183
2184 public boolean isVerified() {
2185 final String fingerprint = this.omemoVerification.getFingerprint();
2186 if (fingerprint == null) {
2187 return false;
2188 }
2189 final FingerprintStatus status =
2190 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2191 return status != null && status.isVerified();
2192 }
2193
2194 public boolean addMedia(final Media media) {
2195 final Set<Media> currentMedia = getMedia();
2196 if (currentMedia.contains(media)) {
2197 throw new IllegalStateException(String.format("%s has already been proposed", media));
2198 }
2199 // TODO add state protection - can only add while ACCEPTED or so
2200 Log.d(Config.LOGTAG, "adding media: " + media);
2201 return webRTCWrapper.addTrack(media);
2202 }
2203
2204 public synchronized void acceptCall() {
2205 switch (this.state) {
2206 case PROPOSED -> {
2207 cancelRingingTimeout();
2208 acceptCallFromProposed();
2209 }
2210 case SESSION_INITIALIZED -> {
2211 cancelRingingTimeout();
2212 acceptCallFromSessionInitialized();
2213 }
2214 case ACCEPTED -> Log.w(
2215 Config.LOGTAG,
2216 id.account.getJid().asBareJid()
2217 + ": the call has already been accepted with another client. UI was just lagging behind");
2218 case PROCEED, SESSION_ACCEPTED -> Log.w(
2219 Config.LOGTAG,
2220 id.account.getJid().asBareJid()
2221 + ": the call has already been accepted. user probably double tapped the UI");
2222 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2223 }
2224 }
2225
2226 public synchronized void rejectCall() {
2227 if (isTerminated()) {
2228 Log.w(
2229 Config.LOGTAG,
2230 id.account.getJid().asBareJid()
2231 + ": received rejectCall() when session has already been terminated. nothing to do");
2232 return;
2233 }
2234 switch (this.state) {
2235 case PROPOSED -> rejectCallFromProposed();
2236 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2237 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2238 }
2239 }
2240
2241 public synchronized void integrationFailure() {
2242 final var state = getState();
2243 if (state == State.PROPOSED) {
2244 Log.e(
2245 Config.LOGTAG,
2246 id.account.getJid().asBareJid()
2247 + ": failed call integration in state proposed");
2248 rejectCallFromProposed();
2249 } else if (state == State.SESSION_INITIALIZED) {
2250 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2251 this.webRTCWrapper.close();
2252 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2253 } else {
2254 throw new IllegalStateException(
2255 String.format("Can not fail integration in state %s", state));
2256 }
2257 }
2258
2259 public synchronized void endCall() {
2260 if (isTerminated()) {
2261 Log.w(
2262 Config.LOGTAG,
2263 id.account.getJid().asBareJid()
2264 + ": received endCall() when session has already been terminated. nothing to do");
2265 return;
2266 }
2267 if (isInState(State.PROPOSED) && isResponder()) {
2268 rejectCallFromProposed();
2269 return;
2270 }
2271 if (isInState(State.PROCEED)) {
2272 if (isInitiator()) {
2273 retractFromProceed();
2274 } else {
2275 rejectCallFromProceed();
2276 }
2277 return;
2278 }
2279 if (isInitiator()
2280 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2281 this.webRTCWrapper.close();
2282 sendSessionTerminate(Reason.CANCEL);
2283 return;
2284 }
2285 if (isInState(State.SESSION_INITIALIZED)) {
2286 rejectCallFromSessionInitiate();
2287 return;
2288 }
2289 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2290 this.webRTCWrapper.close();
2291 sendSessionTerminate(Reason.SUCCESS);
2292 return;
2293 }
2294 if (isInState(
2295 State.TERMINATED_APPLICATION_FAILURE,
2296 State.TERMINATED_CONNECTIVITY_ERROR,
2297 State.TERMINATED_DECLINED_OR_BUSY)) {
2298 Log.d(
2299 Config.LOGTAG,
2300 "ignoring request to end call because already in state " + this.state);
2301 return;
2302 }
2303 throw new IllegalStateException(
2304 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2305 }
2306
2307 private void retractFromProceed() {
2308 Log.d(Config.LOGTAG, "retract from proceed");
2309 this.sendJingleMessage("retract");
2310 closeTransitionLogFinish(State.RETRACTED_RACED);
2311 }
2312
2313 private void closeTransitionLogFinish(final State state) {
2314 this.webRTCWrapper.close();
2315 transitionOrThrow(state);
2316 writeLogMessage(state);
2317 finish();
2318 }
2319
2320 private void setupWebRTC(
2321 final Set<Media> media,
2322 final List<PeerConnection.IceServer> iceServers,
2323 final boolean trickle)
2324 throws WebRTCWrapper.InitializationException {
2325 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2326 this.webRTCWrapper.setup(this.xmppConnectionService);
2327 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2328 this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2329 }
2330
2331 private void acceptCallFromProposed() {
2332 transitionOrThrow(State.PROCEED);
2333 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2334 this.callIntegration.startAudioRouting();
2335 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2336 this.sendJingleMessage("proceed");
2337 }
2338
2339 private void rejectCallFromProposed() {
2340 transitionOrThrow(State.REJECTED);
2341 writeLogMessageMissed();
2342 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2343 this.sendJingleMessage("reject");
2344 finish();
2345 }
2346
2347 private void rejectCallFromProceed() {
2348 this.sendJingleMessage("reject");
2349 closeTransitionLogFinish(State.REJECTED_RACED);
2350 }
2351
2352 private void rejectCallFromSessionInitiate() {
2353 webRTCWrapper.close();
2354 sendSessionTerminate(Reason.DECLINE);
2355 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2356 }
2357
2358 private void sendJingleMessage(final String action) {
2359 sendJingleMessage(action, id.with);
2360 }
2361
2362 private void sendJingleMessage(final String action, final Jid to) {
2363 final MessagePacket messagePacket = new MessagePacket();
2364 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2365 messagePacket.setTo(to);
2366 final Element intent =
2367 messagePacket
2368 .addChild(action, Namespace.JINGLE_MESSAGE)
2369 .setAttribute("id", id.sessionId);
2370 if ("proceed".equals(action)) {
2371 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2372 if (isOmemoEnabled()) {
2373 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2374 final Element device =
2375 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2376 device.setAttribute("id", deviceId);
2377 }
2378 }
2379 messagePacket.addChild("store", "urn:xmpp:hints");
2380 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2381 }
2382
2383 private void sendJingleMessageFinish(final Reason reason) {
2384 final var account = id.getAccount();
2385 final MessagePacket messagePacket =
2386 xmppConnectionService
2387 .getMessageGenerator()
2388 .sessionFinish(id.with, id.sessionId, reason);
2389 xmppConnectionService.sendMessagePacket(account, messagePacket);
2390 }
2391
2392 private boolean isOmemoEnabled() {
2393 final Conversational conversational = message.getConversation();
2394 if (conversational instanceof Conversation) {
2395 return ((Conversation) conversational).getNextEncryption()
2396 == Message.ENCRYPTION_AXOLOTL;
2397 }
2398 return false;
2399 }
2400
2401 private void acceptCallFromSessionInitialized() {
2402 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2403 this.callIntegration.startAudioRouting();
2404 sendSessionAccept();
2405 }
2406
2407 @Override
2408 protected synchronized boolean transition(final State target, final Runnable runnable) {
2409 if (super.transition(target, runnable)) {
2410 updateEndUserState();
2411 updateOngoingCallNotification();
2412 return true;
2413 } else {
2414 return false;
2415 }
2416 }
2417
2418 @Override
2419 public void onIceCandidate(final IceCandidate iceCandidate) {
2420 final RtpContentMap rtpContentMap =
2421 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2422 final IceUdpTransportInfo.Credentials credentials;
2423 try {
2424 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2425 } catch (final IllegalArgumentException e) {
2426 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2427 return;
2428 }
2429 final String uFrag = credentials.ufrag;
2430 final IceUdpTransportInfo.Candidate candidate =
2431 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2432 if (candidate == null) {
2433 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2434 return;
2435 }
2436 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2437 sendTransportInfo(iceCandidate.sdpMid, candidate);
2438 }
2439
2440 @Override
2441 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2442 Log.d(
2443 Config.LOGTAG,
2444 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2445 this.stateHistory.add(newState);
2446 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2447 this.sessionDuration.start();
2448 updateOngoingCallNotification();
2449 } else if (this.sessionDuration.isRunning()) {
2450 this.sessionDuration.stop();
2451 updateOngoingCallNotification();
2452 }
2453
2454 final boolean neverConnected =
2455 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2456
2457 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2458 if (neverConnected) {
2459 if (isTerminated()) {
2460 Log.d(
2461 Config.LOGTAG,
2462 id.account.getJid().asBareJid()
2463 + ": not sending session-terminate after connectivity error because session is already in state "
2464 + this.state);
2465 return;
2466 }
2467 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2468 return;
2469 } else {
2470 this.restartIce();
2471 }
2472 }
2473 updateEndUserState();
2474 }
2475
2476 private void restartIce() {
2477 this.stateHistory.clear();
2478 this.webRTCWrapper.restartIceAsync();
2479 }
2480
2481 @Override
2482 public void onRenegotiationNeeded() {
2483 this.webRTCWrapper.execute(this::renegotiate);
2484 }
2485
2486 private synchronized void renegotiate() {
2487 final SessionDescription sessionDescription;
2488 try {
2489 sessionDescription = setLocalSessionDescription();
2490 } catch (final Exception e) {
2491 final Throwable cause = Throwables.getRootCause(e);
2492 webRTCWrapper.close();
2493 if (isTerminated()) {
2494 Log.d(
2495 Config.LOGTAG,
2496 "failed to renegotiate. session was already terminated",
2497 cause);
2498 return;
2499 }
2500 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2501 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2502 return;
2503 }
2504 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2505 final RtpContentMap currentContentMap = getLocalContentMap();
2506 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2507 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2508
2509 Log.d(
2510 Config.LOGTAG,
2511 id.getAccount().getJid().asBareJid()
2512 + ": renegotiate. iceRestart="
2513 + iceRestart
2514 + " content id diff="
2515 + diff);
2516
2517 if (diff.hasModifications() && iceRestart) {
2518 webRTCWrapper.close();
2519 sendSessionTerminate(
2520 Reason.FAILED_APPLICATION,
2521 "WebRTC unexpectedly tried to modify content and transport at once");
2522 return;
2523 }
2524
2525 if (iceRestart) {
2526 initiateIceRestart(rtpContentMap);
2527 return;
2528 } else if (diff.isEmpty()) {
2529 Log.d(
2530 Config.LOGTAG,
2531 "renegotiation. nothing to do. SignalingState="
2532 + this.webRTCWrapper.getSignalingState());
2533 }
2534
2535 if (diff.added.isEmpty()) {
2536 return;
2537 }
2538 modifyLocalContentMap(rtpContentMap);
2539 sendContentAdd(rtpContentMap, diff.added);
2540 }
2541
2542 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2543 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2544 final JinglePacket jinglePacket =
2545 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2546 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2547 jinglePacket.setTo(id.with);
2548 xmppConnectionService.sendIqPacket(
2549 id.account,
2550 jinglePacket,
2551 (account, response) -> {
2552 if (response.getType() == IqPacket.TYPE.RESULT) {
2553 Log.d(Config.LOGTAG, "received success to our ice restart");
2554 setLocalContentMap(rtpContentMap);
2555 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2556 return;
2557 }
2558 if (response.getType() == IqPacket.TYPE.ERROR) {
2559 if (isTieBreak(response)) {
2560 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2561 return;
2562 }
2563 handleIqErrorResponse(response);
2564 }
2565 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2566 handleIqTimeoutResponse(response);
2567 }
2568 });
2569 }
2570
2571 private boolean isTieBreak(final IqPacket response) {
2572 final Element error = response.findChild("error");
2573 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2574 }
2575
2576 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2577 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2578 this.outgoingContentAdd = contentAdd;
2579 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2580 prepareOutgoingContentMap(contentAdd);
2581 Futures.addCallback(
2582 outgoingContentMapFuture,
2583 new FutureCallback<>() {
2584 @Override
2585 public void onSuccess(final RtpContentMap outgoingContentMap) {
2586 sendContentAdd(outgoingContentMap);
2587 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2588 }
2589
2590 @Override
2591 public void onFailure(@NonNull Throwable throwable) {
2592 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2593 }
2594 },
2595 MoreExecutors.directExecutor());
2596 }
2597
2598 private void sendContentAdd(final RtpContentMap contentAdd) {
2599
2600 final JinglePacket jinglePacket =
2601 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2602 jinglePacket.setTo(id.with);
2603 xmppConnectionService.sendIqPacket(
2604 id.account,
2605 jinglePacket,
2606 (connection, response) -> {
2607 if (response.getType() == IqPacket.TYPE.RESULT) {
2608 Log.d(
2609 Config.LOGTAG,
2610 id.getAccount().getJid().asBareJid()
2611 + ": received ACK to our content-add");
2612 return;
2613 }
2614 if (response.getType() == IqPacket.TYPE.ERROR) {
2615 if (isTieBreak(response)) {
2616 this.outgoingContentAdd = null;
2617 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2618 return;
2619 }
2620 handleIqErrorResponse(response);
2621 }
2622 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2623 handleIqTimeoutResponse(response);
2624 }
2625 });
2626 }
2627
2628 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2629 if (isInitiator()) {
2630 this.initiatorRtpContentMap = rtpContentMap;
2631 } else {
2632 this.responderRtpContentMap = rtpContentMap;
2633 }
2634 }
2635
2636 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2637 if (isInitiator()) {
2638 this.responderRtpContentMap = rtpContentMap;
2639 } else {
2640 this.initiatorRtpContentMap = rtpContentMap;
2641 }
2642 }
2643
2644 // this method is to be used for content map modifications that modify media
2645 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2646 final RtpContentMap activeContents = rtpContentMap.activeContents();
2647 setLocalContentMap(activeContents);
2648 this.callIntegration.setAudioDeviceWhenAvailable(
2649 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2650 updateEndUserState();
2651 }
2652
2653 private SessionDescription setLocalSessionDescription()
2654 throws ExecutionException, InterruptedException {
2655 final org.webrtc.SessionDescription sessionDescription =
2656 this.webRTCWrapper.setLocalDescription(false).get();
2657 return SessionDescription.parse(sessionDescription.description);
2658 }
2659
2660 private void closeWebRTCSessionAfterFailedConnection() {
2661 this.webRTCWrapper.close();
2662 synchronized (this) {
2663 if (isTerminated()) {
2664 Log.d(
2665 Config.LOGTAG,
2666 id.account.getJid().asBareJid()
2667 + ": no need to send session-terminate after failed connection. Other party already did");
2668 return;
2669 }
2670 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2671 }
2672 }
2673
2674 public boolean zeroDuration() {
2675 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2676 }
2677
2678 public long getCallDuration() {
2679 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2680 }
2681
2682 @Override
2683 public CallIntegration getCallIntegration() {
2684 return this.callIntegration;
2685 }
2686
2687 public boolean isMicrophoneEnabled() {
2688 return webRTCWrapper.isMicrophoneEnabled();
2689 }
2690
2691 public boolean setMicrophoneEnabled(final boolean enabled) {
2692 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2693 }
2694
2695 public boolean isVideoEnabled() {
2696 return webRTCWrapper.isVideoEnabled();
2697 }
2698
2699 public void setVideoEnabled(final boolean enabled) {
2700 webRTCWrapper.setVideoEnabled(enabled);
2701 }
2702
2703 public boolean isCameraSwitchable() {
2704 return webRTCWrapper.isCameraSwitchable();
2705 }
2706
2707 public boolean isFrontCamera() {
2708 return webRTCWrapper.isFrontCamera();
2709 }
2710
2711 public ListenableFuture<Boolean> switchCamera() {
2712 return webRTCWrapper.switchCamera();
2713 }
2714
2715 @Override
2716 public synchronized void onCallIntegrationShowIncomingCallUi() {
2717 if (isTerminated()) {
2718 // there might be race conditions with the call integration service invoking this
2719 // callback when the rtp session has already ended.
2720 Log.w(
2721 Config.LOGTAG,
2722 "CallIntegration requested incoming call UI but session was already terminated");
2723 return;
2724 }
2725 // TODO apparently this can be called too early as well?
2726 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2727 }
2728
2729 @Override
2730 public void onCallIntegrationDisconnect() {
2731 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2732 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2733 rejectCall();
2734 } else {
2735 endCall();
2736 }
2737 }
2738
2739 @Override
2740 public void onCallIntegrationReject() {
2741 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2742 try {
2743 rejectCall();
2744 } catch (final IllegalStateException e) {
2745 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2746 }
2747 }
2748
2749 @Override
2750 public void onCallIntegrationAnswer() {
2751 // we need to start the UI to a) show it and b) be able to ask for permissions
2752 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2753 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2754 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2755 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2756 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2757 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2758 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2759 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2760 xmppConnectionService.startActivity(intent);
2761 }
2762
2763 @Override
2764 public void onCallIntegrationSilence() {
2765 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2766 }
2767
2768 @Override
2769 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2770 this.webRTCWrapper.setMicrophoneEnabled(enabled);
2771 }
2772
2773 @Override
2774 public void onAudioDeviceChanged(
2775 final CallIntegration.AudioDevice selectedAudioDevice,
2776 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2777 Log.d(
2778 Config.LOGTAG,
2779 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2780 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2781 selectedAudioDevice, availableAudioDevices);
2782 }
2783
2784 private void updateEndUserState() {
2785 final RtpEndUserState endUserState = getEndUserState();
2786 this.updateCallIntegrationState();
2787 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2788 id.account, id.with, id.sessionId, endUserState);
2789 }
2790
2791 private void updateOngoingCallNotification() {
2792 final State state = this.state;
2793 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2794 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2795 && isResponder()) {
2796 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2797 xmppConnectionService.removeOngoingCall();
2798 return;
2799 }
2800 final boolean reconnecting;
2801 if (state == State.SESSION_ACCEPTED) {
2802 reconnecting =
2803 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2804 } else {
2805 reconnecting = false;
2806 }
2807 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2808 } else {
2809 xmppConnectionService.removeOngoingCall();
2810 }
2811 }
2812
2813 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2814 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2815 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2816 request.setTo(id.account.getDomain());
2817 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2818 xmppConnectionService.sendIqPacket(
2819 id.account,
2820 request,
2821 (account, response) -> {
2822 final var iceServers = IceServers.parse(response);
2823 if (iceServers.isEmpty()) {
2824 Log.w(
2825 Config.LOGTAG,
2826 id.account.getJid().asBareJid()
2827 + ": no ICE server found "
2828 + response);
2829 }
2830 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2831 });
2832 } else {
2833 Log.w(
2834 Config.LOGTAG,
2835 id.account.getJid().asBareJid() + ": has no external service discovery");
2836 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2837 }
2838 }
2839
2840 @Override
2841 protected void terminateTransport() {
2842 this.webRTCWrapper.close();
2843 }
2844
2845 @Override
2846 protected void finish() {
2847 if (isTerminated()) {
2848 this.cancelRingingTimeout();
2849 this.callIntegration.verifyDisconnected();
2850 this.webRTCWrapper.verifyClosed();
2851 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2852 super.finish();
2853 } else {
2854 throw new IllegalStateException(
2855 String.format("Unable to call finish from %s", this.state));
2856 }
2857 }
2858
2859 private void writeLogMessage(final State state) {
2860 final long duration = getCallDuration();
2861 if (state == State.TERMINATED_SUCCESS
2862 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2863 writeLogMessageSuccess(duration);
2864 } else {
2865 writeLogMessageMissed();
2866 }
2867 }
2868
2869 private void writeLogMessageSuccess(final long duration) {
2870 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2871 this.writeMessage();
2872 }
2873
2874 private void writeLogMessageMissed() {
2875 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2876 this.writeMessage();
2877 }
2878
2879 private void writeMessage() {
2880 final Conversational conversational = message.getConversation();
2881 if (conversational instanceof Conversation) {
2882 ((Conversation) conversational).add(this.message);
2883 xmppConnectionService.createMessageAsync(message);
2884 xmppConnectionService.updateConversationUi();
2885 } else {
2886 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2887 }
2888 }
2889
2890 public Optional<VideoTrack> getLocalVideoTrack() {
2891 return webRTCWrapper.getLocalVideoTrack();
2892 }
2893
2894 public Optional<VideoTrack> getRemoteVideoTrack() {
2895 return webRTCWrapper.getRemoteVideoTrack();
2896 }
2897
2898 public EglBase.Context getEglBaseContext() {
2899 return webRTCWrapper.getEglBaseContext();
2900 }
2901
2902 void setProposedMedia(final Set<Media> media) {
2903 this.proposedMedia = media;
2904 this.callIntegration.setVideoState(
2905 Media.audioOnly(media)
2906 ? VideoProfile.STATE_AUDIO_ONLY
2907 : VideoProfile.STATE_BIDIRECTIONAL);
2908 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2909 }
2910
2911 public void fireStateUpdate() {
2912 final RtpEndUserState endUserState = getEndUserState();
2913 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2914 id.account, id.with, id.sessionId, endUserState);
2915 }
2916
2917 public boolean isSwitchToVideoAvailable() {
2918 final boolean prerequisite =
2919 Media.audioOnly(getMedia())
2920 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2921 .contains(getEndUserState());
2922 return prerequisite && remoteHasVideoFeature();
2923 }
2924
2925 private boolean remoteHasVideoFeature() {
2926 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2927 }
2928
2929 private boolean remoteHasSdpOfferAnswer() {
2930 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2931 }
2932
2933 @Override
2934 public Account getAccount() {
2935 return id.account;
2936 }
2937
2938 @Override
2939 public Jid getWith() {
2940 return id.with;
2941 }
2942
2943 @Override
2944 public String getSessionId() {
2945 return id.sessionId;
2946 }
2947
2948 private interface OnIceServersDiscovered {
2949 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2950 }
2951}