1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Account;
34import eu.siacs.conversations.entities.Conversation;
35import eu.siacs.conversations.entities.Conversational;
36import eu.siacs.conversations.entities.Message;
37import eu.siacs.conversations.entities.RtpSessionStatus;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
50import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
51import eu.siacs.conversations.xmpp.stanzas.IqPacket;
52import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
73
74 public static final List<State> STATES_SHOWING_ONGOING_CALL =
75 Arrays.asList(
76 State.PROPOSED,
77 State.PROCEED,
78 State.SESSION_INITIALIZED_PRE_APPROVED,
79 State.SESSION_ACCEPTED);
80 private static final long BUSY_TIME_OUT = 30;
81
82 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
83 private final Queue<
84 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
85 pendingIceCandidates = new LinkedList<>();
86 private final OmemoVerification omemoVerification = new OmemoVerification();
87 private final CallIntegration callIntegration;
88 private final Message message;
89
90 private Set<Media> proposedMedia;
91 private RtpContentMap initiatorRtpContentMap;
92 private RtpContentMap responderRtpContentMap;
93 private RtpContentMap incomingContentAdd;
94 private RtpContentMap outgoingContentAdd;
95 private IceUdpTransportInfo.Setup peerDtlsSetup;
96 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
97 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
98 private ScheduledFuture<?> ringingTimeoutFuture;
99
100 JingleRtpConnection(
101 final JingleConnectionManager jingleConnectionManager,
102 final Id id,
103 final Jid initiator) {
104 this(
105 jingleConnectionManager,
106 id,
107 initiator,
108 new CallIntegration(
109 jingleConnectionManager
110 .getXmppConnectionService()
111 .getApplicationContext()));
112 this.callIntegration.setAddress(
113 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
114 this.callIntegration.setInitialized();
115 }
116
117 JingleRtpConnection(
118 final JingleConnectionManager jingleConnectionManager,
119 final Id id,
120 final Jid initiator,
121 final CallIntegration callIntegration) {
122 super(jingleConnectionManager, id, initiator);
123 final Conversation conversation =
124 jingleConnectionManager
125 .getXmppConnectionService()
126 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
127 this.message =
128 new Message(
129 conversation,
130 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
131 Message.TYPE_RTP_SESSION,
132 id.sessionId);
133 this.callIntegration = callIntegration;
134 this.callIntegration.setCallback(this);
135 }
136
137 @Override
138 synchronized void deliverPacket(final JinglePacket jinglePacket) {
139 switch (jinglePacket.getAction()) {
140 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
141 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
142 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
143 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
144 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
145 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
146 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
147 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
148 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
149 default -> {
150 respondOk(jinglePacket);
151 Log.d(
152 Config.LOGTAG,
153 String.format(
154 "%s: received unhandled jingle action %s",
155 id.account.getJid().asBareJid(), jinglePacket.getAction()));
156 }
157 }
158 }
159
160 @Override
161 synchronized void notifyRebound() {
162 if (isTerminated()) {
163 return;
164 }
165 webRTCWrapper.close();
166 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
167 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
168 }
169 if (isInState(
170 State.SESSION_INITIALIZED,
171 State.SESSION_INITIALIZED_PRE_APPROVED,
172 State.SESSION_ACCEPTED)) {
173 // we might have already changed resources (full jid) at this point; so this might not
174 // even reach the other party
175 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
176 } else {
177 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
178 finish();
179 }
180 }
181
182 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
183 respondOk(jinglePacket);
184 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
185 final State previous = this.state;
186 Log.d(
187 Config.LOGTAG,
188 id.account.getJid().asBareJid()
189 + ": received session terminate reason="
190 + wrapper.reason
191 + "("
192 + Strings.nullToEmpty(wrapper.text)
193 + ") while in state "
194 + previous);
195 if (TERMINATED.contains(previous)) {
196 Log.d(
197 Config.LOGTAG,
198 id.account.getJid().asBareJid()
199 + ": ignoring session terminate because already in "
200 + previous);
201 return;
202 }
203 webRTCWrapper.close();
204 final State target = reasonToState(wrapper.reason);
205 transitionOrThrow(target);
206 writeLogMessage(target);
207 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
208 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
209 }
210 finish();
211 }
212
213 private void receiveTransportInfo(final JinglePacket jinglePacket) {
214 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
215 // INITIALIZED only after transport-info has been received
216 if (isInState(
217 State.NULL,
218 State.PROCEED,
219 State.SESSION_INITIALIZED,
220 State.SESSION_INITIALIZED_PRE_APPROVED,
221 State.SESSION_ACCEPTED)) {
222 final RtpContentMap contentMap;
223 try {
224 contentMap = RtpContentMap.of(jinglePacket);
225 } catch (final IllegalArgumentException | NullPointerException e) {
226 Log.d(
227 Config.LOGTAG,
228 id.account.getJid().asBareJid()
229 + ": improperly formatted contents; ignoring",
230 e);
231 respondOk(jinglePacket);
232 return;
233 }
234 receiveTransportInfo(jinglePacket, contentMap);
235 } else {
236 if (isTerminated()) {
237 respondOk(jinglePacket);
238 Log.d(
239 Config.LOGTAG,
240 id.account.getJid().asBareJid()
241 + ": ignoring out-of-order transport info; we where already terminated");
242 } else {
243 Log.d(
244 Config.LOGTAG,
245 id.account.getJid().asBareJid()
246 + ": received transport info while in state="
247 + this.state);
248 terminateWithOutOfOrder(jinglePacket);
249 }
250 }
251 }
252
253 private void receiveTransportInfo(
254 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
255 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
256 candidates = contentMap.contents.entrySet();
257 final RtpContentMap remote = getRemoteContentMap();
258 final Set<String> remoteContentIds =
259 remote == null ? Collections.emptySet() : remote.contents.keySet();
260 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
261 Log.d(
262 Config.LOGTAG,
263 "received transport-info for unknown contents "
264 + contentMap.contents.keySet()
265 + " (known: "
266 + remoteContentIds
267 + ")");
268 respondOk(jinglePacket);
269 pendingIceCandidates.addAll(candidates);
270 return;
271 }
272 if (this.state != State.SESSION_ACCEPTED) {
273 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
274 respondOk(jinglePacket);
275 pendingIceCandidates.addAll(candidates);
276 return;
277 }
278 // zero candidates + modified credentials are an ICE restart offer
279 if (checkForIceRestart(jinglePacket, contentMap)) {
280 return;
281 }
282 respondOk(jinglePacket);
283 try {
284 processCandidates(candidates);
285 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
286 Log.w(
287 Config.LOGTAG,
288 id.account.getJid().asBareJid()
289 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
290 }
291 }
292
293 private void receiveContentAdd(final JinglePacket jinglePacket) {
294 final RtpContentMap modification;
295 try {
296 modification = RtpContentMap.of(jinglePacket);
297 modification.requireContentDescriptions();
298 } catch (final RuntimeException e) {
299 Log.d(
300 Config.LOGTAG,
301 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
302 Throwables.getRootCause(e));
303 respondOk(jinglePacket);
304 webRTCWrapper.close();
305 sendSessionTerminate(Reason.of(e), e.getMessage());
306 return;
307 }
308 if (isInState(State.SESSION_ACCEPTED)) {
309 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
310 final ListenableFuture<RtpContentMap> future =
311 receiveRtpContentMap(
312 modification,
313 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
314 Futures.addCallback(
315 future,
316 new FutureCallback<>() {
317 @Override
318 public void onSuccess(final RtpContentMap rtpContentMap) {
319 receiveContentAdd(jinglePacket, rtpContentMap);
320 }
321
322 @Override
323 public void onFailure(@NonNull Throwable throwable) {
324 respondOk(jinglePacket);
325 final Throwable rootCause = Throwables.getRootCause(throwable);
326 Log.d(
327 Config.LOGTAG,
328 id.account.getJid().asBareJid()
329 + ": improperly formatted contents in content-add",
330 throwable);
331 webRTCWrapper.close();
332 sendSessionTerminate(
333 Reason.ofThrowable(rootCause), rootCause.getMessage());
334 }
335 },
336 MoreExecutors.directExecutor());
337 } else {
338 terminateWithOutOfOrder(jinglePacket);
339 }
340 }
341
342 private void receiveContentAdd(
343 final JinglePacket jinglePacket, final RtpContentMap modification) {
344 final RtpContentMap remote = getRemoteContentMap();
345 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
346 respondOk(jinglePacket);
347 this.webRTCWrapper.close();
348 sendSessionTerminate(
349 Reason.FAILED_APPLICATION,
350 String.format(
351 "contents with names %s already exists",
352 Joiner.on(", ").join(modification.getNames())));
353 return;
354 }
355 final ContentAddition contentAddition =
356 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
357
358 final RtpContentMap outgoing = this.outgoingContentAdd;
359 final Set<ContentAddition.Summary> outgoingContentAddSummary =
360 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
361
362 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
363 if (isInitiator()) {
364 Log.d(
365 Config.LOGTAG,
366 id.getAccount().getJid().asBareJid()
367 + ": respond with tie break to matching content-add offer");
368 respondWithTieBreak(jinglePacket);
369 } else {
370 Log.d(
371 Config.LOGTAG,
372 id.getAccount().getJid().asBareJid()
373 + ": automatically accept matching content-add offer");
374 acceptContentAdd(contentAddition.summary, modification);
375 }
376 return;
377 }
378
379 // once we can display multiple video tracks we can be more loose with this condition
380 // theoretically it should also be fine to automatically accept audio only contents
381 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
382 Log.d(
383 Config.LOGTAG,
384 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
385 this.incomingContentAdd = modification;
386 respondOk(jinglePacket);
387 updateEndUserState();
388 } else {
389 respondOk(jinglePacket);
390 // TODO do we want to add a reason?
391 rejectContentAdd(modification);
392 }
393 }
394
395 private void receiveContentAccept(final JinglePacket jinglePacket) {
396 final RtpContentMap receivedContentAccept;
397 try {
398 receivedContentAccept = RtpContentMap.of(jinglePacket);
399 receivedContentAccept.requireContentDescriptions();
400 } catch (final RuntimeException e) {
401 Log.d(
402 Config.LOGTAG,
403 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
404 Throwables.getRootCause(e));
405 respondOk(jinglePacket);
406 webRTCWrapper.close();
407 sendSessionTerminate(Reason.of(e), e.getMessage());
408 return;
409 }
410
411 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
412 if (outgoingContentAdd == null) {
413 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
414 terminateWithOutOfOrder(jinglePacket);
415 return;
416 }
417 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
418 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
419 this.outgoingContentAdd = null;
420 respondOk(jinglePacket);
421 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
422 final ListenableFuture<RtpContentMap> future =
423 receiveRtpContentMap(
424 receivedContentAccept,
425 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
426 Futures.addCallback(
427 future,
428 new FutureCallback<>() {
429 @Override
430 public void onSuccess(final RtpContentMap result) {
431 receiveContentAccept(result);
432 }
433
434 @Override
435 public void onFailure(@NonNull final Throwable throwable) {
436 webRTCWrapper.close();
437 sendSessionTerminate(
438 Reason.ofThrowable(throwable), throwable.getMessage());
439 }
440 },
441 MoreExecutors.directExecutor());
442 } else {
443 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
444 terminateWithOutOfOrder(jinglePacket);
445 }
446 }
447
448 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
449 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
450 final RtpContentMap modifiedContentMap =
451 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
452
453 setRemoteContentMap(modifiedContentMap);
454
455 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
456
457 final org.webrtc.SessionDescription sdp =
458 new org.webrtc.SessionDescription(
459 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
460
461 try {
462 this.webRTCWrapper.setRemoteDescription(sdp).get();
463 } catch (final Exception e) {
464 final Throwable cause = Throwables.getRootCause(e);
465 Log.d(
466 Config.LOGTAG,
467 id.getAccount().getJid().asBareJid()
468 + ": unable to set remote description after receiving content-accept",
469 cause);
470 webRTCWrapper.close();
471 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
472 return;
473 }
474 Log.d(
475 Config.LOGTAG,
476 id.getAccount().getJid().asBareJid()
477 + ": remote has accepted content-add "
478 + ContentAddition.summary(receivedContentAccept));
479 processCandidates(receivedContentAccept.contents.entrySet());
480 updateEndUserState();
481 }
482
483 private void receiveContentModify(final JinglePacket jinglePacket) {
484 if (this.state != State.SESSION_ACCEPTED) {
485 terminateWithOutOfOrder(jinglePacket);
486 return;
487 }
488 final Map<String, Content.Senders> modification =
489 Maps.transformEntries(
490 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
491 final boolean isInitiator = isInitiator();
492 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
493 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
494 final Set<String> currentOutgoingMediaIds =
495 currentOutgoing == null
496 ? Collections.emptySet()
497 : currentOutgoing.contents.keySet();
498 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
499 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
500 respondOk(jinglePacket);
501 final RtpContentMap modifiedContentMap;
502 try {
503 modifiedContentMap =
504 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
505 } catch (final IllegalArgumentException e) {
506 webRTCWrapper.close();
507 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
508 return;
509 }
510 this.outgoingContentAdd = modifiedContentMap;
511 Log.d(
512 Config.LOGTAG,
513 id.account.getJid().asBareJid()
514 + ": processed content-modification for pending content-add");
515 } else if (remoteContentMap != null
516 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
517 respondOk(jinglePacket);
518 final RtpContentMap modifiedRemoteContentMap;
519 try {
520 modifiedRemoteContentMap =
521 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
522 } catch (final IllegalArgumentException e) {
523 webRTCWrapper.close();
524 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
525 return;
526 }
527 final SessionDescription offer;
528 try {
529 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
530 } catch (final IllegalArgumentException | NullPointerException e) {
531 Log.d(
532 Config.LOGTAG,
533 id.getAccount().getJid().asBareJid()
534 + ": unable convert offer from content-modify to SDP",
535 e);
536 webRTCWrapper.close();
537 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
538 return;
539 }
540 Log.d(
541 Config.LOGTAG,
542 id.account.getJid().asBareJid() + ": auto accepting content-modification");
543 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
544 } else {
545 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
546 respondWithItemNotFound(jinglePacket);
547 }
548 }
549
550 private void autoAcceptContentModify(
551 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
552 this.setRemoteContentMap(modifiedRemoteContentMap);
553 final org.webrtc.SessionDescription sdp =
554 new org.webrtc.SessionDescription(
555 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
556 try {
557 this.webRTCWrapper.setRemoteDescription(sdp).get();
558 // auto accept is only done when we already have tracks
559 final SessionDescription answer = setLocalSessionDescription();
560 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
561 modifyLocalContentMap(rtpContentMap);
562 // we do not need to send an answer but do we have to resend the candidates currently in
563 // SDP?
564 // resendCandidatesFromSdp(answer);
565 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
566 } catch (final Exception e) {
567 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
568 webRTCWrapper.close();
569 sendSessionTerminate(Reason.FAILED_APPLICATION);
570 }
571 }
572
573 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
574 final SessionDescription answer) {
575 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
576 new ImmutableMultimap.Builder<>();
577 for (final SessionDescription.Media media : answer.media) {
578 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
579 if (Strings.isNullOrEmpty(mid)) {
580 continue;
581 }
582 for (final String sdpCandidate : media.attributes.get("candidate")) {
583 final IceUdpTransportInfo.Candidate candidate =
584 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
585 if (candidate != null) {
586 candidateBuilder.put(mid, candidate);
587 }
588 }
589 }
590 return candidateBuilder.build();
591 }
592
593 private void receiveContentReject(final JinglePacket jinglePacket) {
594 final RtpContentMap receivedContentReject;
595 try {
596 receivedContentReject = RtpContentMap.of(jinglePacket);
597 } catch (final RuntimeException e) {
598 Log.d(
599 Config.LOGTAG,
600 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
601 Throwables.getRootCause(e));
602 respondOk(jinglePacket);
603 this.webRTCWrapper.close();
604 sendSessionTerminate(Reason.of(e), e.getMessage());
605 return;
606 }
607
608 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
609 if (outgoingContentAdd == null) {
610 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
611 terminateWithOutOfOrder(jinglePacket);
612 return;
613 }
614 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
615 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
616 this.outgoingContentAdd = null;
617 respondOk(jinglePacket);
618 Log.d(Config.LOGTAG, jinglePacket.toString());
619 receiveContentReject(ourSummary);
620 } else {
621 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
622 terminateWithOutOfOrder(jinglePacket);
623 }
624 }
625
626 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
627 try {
628 this.webRTCWrapper.removeTrack(Media.VIDEO);
629 final RtpContentMap localContentMap = customRollback();
630 modifyLocalContentMap(localContentMap);
631 } catch (final Exception e) {
632 final Throwable cause = Throwables.getRootCause(e);
633 Log.d(
634 Config.LOGTAG,
635 id.getAccount().getJid().asBareJid()
636 + ": unable to rollback local description after receiving content-reject",
637 cause);
638 webRTCWrapper.close();
639 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
640 return;
641 }
642 Log.d(
643 Config.LOGTAG,
644 id.getAccount().getJid().asBareJid()
645 + ": remote has rejected our content-add "
646 + summary);
647 }
648
649 private void receiveContentRemove(final JinglePacket jinglePacket) {
650 final RtpContentMap receivedContentRemove;
651 try {
652 receivedContentRemove = RtpContentMap.of(jinglePacket);
653 receivedContentRemove.requireContentDescriptions();
654 } catch (final RuntimeException e) {
655 Log.d(
656 Config.LOGTAG,
657 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
658 Throwables.getRootCause(e));
659 respondOk(jinglePacket);
660 this.webRTCWrapper.close();
661 sendSessionTerminate(Reason.of(e), e.getMessage());
662 return;
663 }
664 respondOk(jinglePacket);
665 receiveContentRemove(receivedContentRemove);
666 }
667
668 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
669 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
670 final Set<ContentAddition.Summary> contentAddSummary =
671 incomingContentAdd == null
672 ? Collections.emptySet()
673 : ContentAddition.summary(incomingContentAdd);
674 final Set<ContentAddition.Summary> removeSummary =
675 ContentAddition.summary(receivedContentRemove);
676 if (contentAddSummary.equals(removeSummary)) {
677 this.incomingContentAdd = null;
678 updateEndUserState();
679 } else {
680 webRTCWrapper.close();
681 sendSessionTerminate(
682 Reason.FAILED_APPLICATION,
683 String.format(
684 "%s only supports %s as a means to retract a not yet accepted %s",
685 BuildConfig.APP_NAME,
686 JinglePacket.Action.CONTENT_REMOVE,
687 JinglePacket.Action.CONTENT_ADD));
688 }
689 }
690
691 public synchronized void retractContentAdd() {
692 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
693 if (outgoingContentAdd == null) {
694 throw new IllegalStateException("Not outgoing content add");
695 }
696 try {
697 webRTCWrapper.removeTrack(Media.VIDEO);
698 final RtpContentMap localContentMap = customRollback();
699 modifyLocalContentMap(localContentMap);
700 } catch (final Exception e) {
701 final Throwable cause = Throwables.getRootCause(e);
702 Log.d(
703 Config.LOGTAG,
704 id.getAccount().getJid().asBareJid()
705 + ": unable to rollback local description after trying to retract content-add",
706 cause);
707 webRTCWrapper.close();
708 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
709 return;
710 }
711 this.outgoingContentAdd = null;
712 final JinglePacket retract =
713 outgoingContentAdd
714 .toStub()
715 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
716 this.send(retract);
717 Log.d(
718 Config.LOGTAG,
719 id.getAccount().getJid()
720 + ": retract content-add "
721 + ContentAddition.summary(outgoingContentAdd));
722 }
723
724 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
725 final SessionDescription sdp = setLocalSessionDescription();
726 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
727 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
728 this.webRTCWrapper
729 .setRemoteDescription(
730 new org.webrtc.SessionDescription(
731 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
732 .get();
733 return localRtpContentMap;
734 }
735
736 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
737 final RtpContentMap currentRemote = getRemoteContentMap();
738 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
739 if (diff.isEmpty()) {
740 throw new IllegalStateException(
741 "Unexpected rollback condition. No difference between local and remote");
742 }
743 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
744 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
745 final RtpContentMap nextRemote =
746 currentRemote.addContent(
747 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
748 return SessionDescription.of(nextRemote, isResponder());
749 }
750 throw new IllegalStateException(
751 "Unexpected rollback condition. Senders were not uniformly none");
752 }
753
754 public synchronized void acceptContentAdd(
755 @NonNull final Set<ContentAddition.Summary> contentAddition) {
756 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
757 if (incomingContentAdd == null) {
758 throw new IllegalStateException("No incoming content add");
759 }
760
761 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
762 this.incomingContentAdd = null;
763 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
764 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
765 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
766 Log.d(
767 Config.LOGTAG,
768 "content addition is receive only. we want to upgrade to 'both'");
769 final RtpContentMap modifiedSenders =
770 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
771 final JinglePacket proposedContentModification =
772 modifiedSenders
773 .toStub()
774 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
775 proposedContentModification.setTo(id.with);
776 xmppConnectionService.sendIqPacket(
777 id.account,
778 proposedContentModification,
779 (account, response) -> {
780 if (response.getType() == IqPacket.TYPE.RESULT) {
781 Log.d(
782 Config.LOGTAG,
783 id.account.getJid().asBareJid()
784 + ": remote has accepted our upgrade to senders=both");
785 acceptContentAdd(
786 ContentAddition.summary(modifiedSenders), modifiedSenders);
787 } else {
788 Log.d(
789 Config.LOGTAG,
790 id.account.getJid().asBareJid()
791 + ": remote has rejected our upgrade to senders=both");
792 acceptContentAdd(contentAddition, incomingContentAdd);
793 }
794 });
795 } else {
796 acceptContentAdd(contentAddition, incomingContentAdd);
797 }
798 } else {
799 throw new IllegalStateException(
800 "Accepted content add does not match pending content-add");
801 }
802 }
803
804 private void acceptContentAdd(
805 @NonNull final Set<ContentAddition.Summary> contentAddition,
806 final RtpContentMap incomingContentAdd) {
807 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
808 final RtpContentMap modifiedContentMap =
809 getRemoteContentMap().addContent(incomingContentAdd, setup);
810 this.setRemoteContentMap(modifiedContentMap);
811
812 final SessionDescription offer;
813 try {
814 offer = SessionDescription.of(modifiedContentMap, isResponder());
815 } catch (final IllegalArgumentException | NullPointerException e) {
816 Log.d(
817 Config.LOGTAG,
818 id.getAccount().getJid().asBareJid()
819 + ": unable convert offer from content-add to SDP",
820 e);
821 webRTCWrapper.close();
822 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
823 return;
824 }
825 this.incomingContentAdd = null;
826 acceptContentAdd(contentAddition, offer);
827 }
828
829 private void acceptContentAdd(
830 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
831 final org.webrtc.SessionDescription sdp =
832 new org.webrtc.SessionDescription(
833 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
834 try {
835 this.webRTCWrapper.setRemoteDescription(sdp).get();
836
837 // TODO add tracks for 'media' where contentAddition.senders matches
838
839 // TODO if senders.sending(isInitiator())
840
841 this.webRTCWrapper.addTrack(Media.VIDEO);
842
843 // TODO add additional transceivers for recv only cases
844
845 final SessionDescription answer = setLocalSessionDescription();
846 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
847
848 final RtpContentMap contentAcceptMap =
849 rtpContentMap.toContentModification(
850 Collections2.transform(contentAddition, ca -> ca.name));
851
852 Log.d(
853 Config.LOGTAG,
854 id.getAccount().getJid().asBareJid()
855 + ": sending content-accept "
856 + ContentAddition.summary(contentAcceptMap));
857
858 addIceCandidatesFromBlackLog();
859
860 modifyLocalContentMap(rtpContentMap);
861 final ListenableFuture<RtpContentMap> future =
862 prepareOutgoingContentMap(contentAcceptMap);
863 Futures.addCallback(
864 future,
865 new FutureCallback<>() {
866 @Override
867 public void onSuccess(final RtpContentMap rtpContentMap) {
868 sendContentAccept(rtpContentMap);
869 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
870 }
871
872 @Override
873 public void onFailure(@NonNull final Throwable throwable) {
874 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
875 }
876 },
877 MoreExecutors.directExecutor());
878 } catch (final Exception e) {
879 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
880 webRTCWrapper.close();
881 sendSessionTerminate(Reason.FAILED_APPLICATION);
882 }
883 }
884
885 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
886 final JinglePacket jinglePacket =
887 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
888 send(jinglePacket);
889 }
890
891 public synchronized void rejectContentAdd() {
892 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
893 if (incomingContentAdd == null) {
894 throw new IllegalStateException("No incoming content add");
895 }
896 this.incomingContentAdd = null;
897 updateEndUserState();
898 rejectContentAdd(incomingContentAdd);
899 }
900
901 private void rejectContentAdd(final RtpContentMap contentMap) {
902 final JinglePacket jinglePacket =
903 contentMap
904 .toStub()
905 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
906 Log.d(
907 Config.LOGTAG,
908 id.getAccount().getJid().asBareJid()
909 + ": rejecting content "
910 + ContentAddition.summary(contentMap));
911 send(jinglePacket);
912 }
913
914 private boolean checkForIceRestart(
915 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
916 final RtpContentMap existing = getRemoteContentMap();
917 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
918 final IceUdpTransportInfo.Credentials newCredentials;
919 try {
920 existingCredentials = existing.getCredentials();
921 newCredentials = rtpContentMap.getDistinctCredentials();
922 } catch (final IllegalStateException e) {
923 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
924 return false;
925 }
926 if (existingCredentials.contains(newCredentials)) {
927 return false;
928 }
929 // TODO an alternative approach is to check if we already got an iq result to our
930 // ICE-restart
931 // and if that's the case we are seeing an answer.
932 // This might be more spec compliant but also more error prone potentially
933 final boolean isSignalStateStable =
934 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
935 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
936 final boolean isOffer = rtpContentMap.emptyCandidates();
937 final RtpContentMap restartContentMap;
938 try {
939 if (isOffer) {
940 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
941 restartContentMap =
942 existing.modifiedCredentials(
943 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
944 } else {
945 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
946 Log.d(
947 Config.LOGTAG,
948 "received confirmation of ICE restart"
949 + newCredentials
950 + " peer_setup="
951 + setup);
952 // DTLS setup attribute needs to be rewritten to reflect current peer state
953 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
954 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
955 }
956 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
957 return isOffer;
958 } else {
959 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
960 respondWithTieBreak(jinglePacket);
961 return true;
962 }
963 } catch (final Exception exception) {
964 respondOk(jinglePacket);
965 final Throwable rootCause = Throwables.getRootCause(exception);
966 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
967 // If this happens a termination is already in progress
968 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
969 return true;
970 }
971 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
972 webRTCWrapper.close();
973 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
974 return true;
975 }
976 }
977
978 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
979 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
980 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
981 throw new IllegalStateException("Invalid peer setup");
982 }
983 return peerSetup;
984 }
985
986 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
987 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
988 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
989 }
990 this.peerDtlsSetup = setup;
991 }
992
993 private boolean applyIceRestart(
994 final JinglePacket jinglePacket,
995 final RtpContentMap restartContentMap,
996 final boolean isOffer)
997 throws ExecutionException, InterruptedException {
998 final SessionDescription sessionDescription =
999 SessionDescription.of(restartContentMap, isResponder());
1000 final org.webrtc.SessionDescription.Type type =
1001 isOffer
1002 ? org.webrtc.SessionDescription.Type.OFFER
1003 : org.webrtc.SessionDescription.Type.ANSWER;
1004 org.webrtc.SessionDescription sdp =
1005 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1006 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1007 if (isInitiator()) {
1008 // We ignore the offer and respond with tie-break. This will clause the responder
1009 // not to apply the content map
1010 return false;
1011 }
1012 }
1013 webRTCWrapper.setRemoteDescription(sdp).get();
1014 setRemoteContentMap(restartContentMap);
1015 if (isOffer) {
1016 final SessionDescription localSessionDescription = setLocalSessionDescription();
1017 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1018 // We need to respond OK before sending any candidates
1019 respondOk(jinglePacket);
1020 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1021 } else {
1022 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1023 }
1024 return true;
1025 }
1026
1027 private void processCandidates(
1028 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1029 contents) {
1030 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1031 content : contents) {
1032 processCandidate(content);
1033 }
1034 }
1035
1036 private void processCandidate(
1037 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1038 content) {
1039 final RtpContentMap rtpContentMap = getRemoteContentMap();
1040 final List<String> indices = toIdentificationTags(rtpContentMap);
1041 final String sdpMid = content.getKey(); // aka content name
1042 final IceUdpTransportInfo transport = content.getValue().transport;
1043 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1044
1045 // TODO check that credentials remained the same
1046
1047 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1048 final String sdp;
1049 try {
1050 sdp = candidate.toSdpAttribute(credentials.ufrag);
1051 } catch (final IllegalArgumentException e) {
1052 Log.d(
1053 Config.LOGTAG,
1054 id.account.getJid().asBareJid()
1055 + ": ignoring invalid ICE candidate "
1056 + e.getMessage());
1057 continue;
1058 }
1059 final int mLineIndex = indices.indexOf(sdpMid);
1060 if (mLineIndex < 0) {
1061 Log.w(
1062 Config.LOGTAG,
1063 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1064 }
1065 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1066 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1067 this.webRTCWrapper.addIceCandidate(iceCandidate);
1068 }
1069 }
1070
1071 private RtpContentMap getRemoteContentMap() {
1072 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1073 }
1074
1075 private RtpContentMap getLocalContentMap() {
1076 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1077 }
1078
1079 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1080 final Group originalGroup = rtpContentMap.group;
1081 final List<String> identificationTags =
1082 originalGroup == null
1083 ? rtpContentMap.getNames()
1084 : originalGroup.getIdentificationTags();
1085 if (identificationTags.size() == 0) {
1086 Log.w(
1087 Config.LOGTAG,
1088 id.account.getJid().asBareJid()
1089 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1090 }
1091 return identificationTags;
1092 }
1093
1094 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1095 final JinglePacket jinglePacket, final boolean expectVerification) {
1096 try {
1097 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1098 } catch (final Exception e) {
1099 return Futures.immediateFailedFuture(e);
1100 }
1101 }
1102
1103 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1104 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1105 Log.d(
1106 Config.LOGTAG,
1107 "receiveRtpContentMap("
1108 + receivedContentMap.getClass().getSimpleName()
1109 + ",expectVerification="
1110 + expectVerification
1111 + ")");
1112 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1113 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1114 id.account
1115 .getAxolotlService()
1116 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1117 return Futures.transform(
1118 future,
1119 omemoVerifiedPayload -> {
1120 // TODO test if an exception here triggers a correct abort
1121 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1122 Log.d(
1123 Config.LOGTAG,
1124 id.account.getJid().asBareJid()
1125 + ": received verifiable DTLS fingerprint via "
1126 + omemoVerification);
1127 return omemoVerifiedPayload.getPayload();
1128 },
1129 MoreExecutors.directExecutor());
1130 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1131 return Futures.immediateFailedFuture(
1132 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1133 } else {
1134 return Futures.immediateFuture(receivedContentMap);
1135 }
1136 }
1137
1138 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1139 if (isInitiator()) {
1140 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1141 return;
1142 }
1143 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1144 Futures.addCallback(
1145 future,
1146 new FutureCallback<>() {
1147 @Override
1148 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1149 receiveSessionInitiate(jinglePacket, rtpContentMap);
1150 }
1151
1152 @Override
1153 public void onFailure(@NonNull final Throwable throwable) {
1154 respondOk(jinglePacket);
1155 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1156 }
1157 },
1158 MoreExecutors.directExecutor());
1159 }
1160
1161 private void receiveSessionInitiate(
1162 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1163 try {
1164 contentMap.requireContentDescriptions();
1165 contentMap.requireDTLSFingerprint(true);
1166 } catch (final RuntimeException e) {
1167 Log.d(
1168 Config.LOGTAG,
1169 id.account.getJid().asBareJid() + ": improperly formatted contents",
1170 Throwables.getRootCause(e));
1171 respondOk(jinglePacket);
1172 sendSessionTerminate(Reason.of(e), e.getMessage());
1173 return;
1174 }
1175 Log.d(
1176 Config.LOGTAG,
1177 "processing session-init with " + contentMap.contents.size() + " contents");
1178 final State target;
1179 if (this.state == State.PROCEED) {
1180 Preconditions.checkState(
1181 proposedMedia != null && proposedMedia.size() > 0,
1182 "proposed media must be set when processing pre-approved session-initiate");
1183 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1184 sendSessionTerminate(
1185 Reason.SECURITY_ERROR,
1186 String.format(
1187 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1188 this.proposedMedia, contentMap.getMedia()));
1189 return;
1190 }
1191 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1192 } else {
1193 target = State.SESSION_INITIALIZED;
1194 setProposedMedia(contentMap.getMedia());
1195 }
1196 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1197 respondOk(jinglePacket);
1198 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1199 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1200 Log.d(
1201 Config.LOGTAG,
1202 id.account.getJid().asBareJid()
1203 + ": automatically accepting session-initiate");
1204 sendSessionAccept();
1205 } else {
1206 Log.d(
1207 Config.LOGTAG,
1208 id.account.getJid().asBareJid()
1209 + ": received not pre-approved session-initiate. start ringing");
1210 startRinging();
1211 }
1212 } else {
1213 Log.d(
1214 Config.LOGTAG,
1215 String.format(
1216 "%s: received session-initiate while in state %s",
1217 id.account.getJid().asBareJid(), state));
1218 terminateWithOutOfOrder(jinglePacket);
1219 }
1220 }
1221
1222 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1223 if (isResponder()) {
1224 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1225 return;
1226 }
1227 final ListenableFuture<RtpContentMap> future =
1228 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1229 Futures.addCallback(
1230 future,
1231 new FutureCallback<>() {
1232 @Override
1233 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1234 receiveSessionAccept(jinglePacket, rtpContentMap);
1235 }
1236
1237 @Override
1238 public void onFailure(@NonNull final Throwable throwable) {
1239 respondOk(jinglePacket);
1240 Log.d(
1241 Config.LOGTAG,
1242 id.account.getJid().asBareJid()
1243 + ": improperly formatted contents in session-accept",
1244 throwable);
1245 webRTCWrapper.close();
1246 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1247 }
1248 },
1249 MoreExecutors.directExecutor());
1250 }
1251
1252 private void receiveSessionAccept(
1253 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1254 try {
1255 contentMap.requireContentDescriptions();
1256 contentMap.requireDTLSFingerprint();
1257 } catch (final RuntimeException e) {
1258 respondOk(jinglePacket);
1259 Log.d(
1260 Config.LOGTAG,
1261 id.account.getJid().asBareJid()
1262 + ": improperly formatted contents in session-accept",
1263 e);
1264 webRTCWrapper.close();
1265 sendSessionTerminate(Reason.of(e), e.getMessage());
1266 return;
1267 }
1268 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1269 if (!initiatorMedia.equals(contentMap.getMedia())) {
1270 sendSessionTerminate(
1271 Reason.SECURITY_ERROR,
1272 String.format(
1273 "Your session-included included media %s but our session-initiate was %s",
1274 this.proposedMedia, contentMap.getMedia()));
1275 return;
1276 }
1277 Log.d(
1278 Config.LOGTAG,
1279 "processing session-accept with " + contentMap.contents.size() + " contents");
1280 if (transition(State.SESSION_ACCEPTED)) {
1281 respondOk(jinglePacket);
1282 receiveSessionAccept(contentMap);
1283 } else {
1284 Log.d(
1285 Config.LOGTAG,
1286 String.format(
1287 "%s: received session-accept while in state %s",
1288 id.account.getJid().asBareJid(), state));
1289 respondOk(jinglePacket);
1290 }
1291 }
1292
1293 private void receiveSessionAccept(final RtpContentMap contentMap) {
1294 this.responderRtpContentMap = contentMap;
1295 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1296 final SessionDescription sessionDescription;
1297 try {
1298 sessionDescription = SessionDescription.of(contentMap, false);
1299 } catch (final IllegalArgumentException | NullPointerException e) {
1300 Log.d(
1301 Config.LOGTAG,
1302 id.account.getJid().asBareJid()
1303 + ": unable convert offer from session-accept to SDP",
1304 e);
1305 webRTCWrapper.close();
1306 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1307 return;
1308 }
1309 final org.webrtc.SessionDescription answer =
1310 new org.webrtc.SessionDescription(
1311 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1312 try {
1313 this.webRTCWrapper.setRemoteDescription(answer).get();
1314 } catch (final Exception e) {
1315 Log.d(
1316 Config.LOGTAG,
1317 id.account.getJid().asBareJid()
1318 + ": unable to set remote description after receiving session-accept",
1319 Throwables.getRootCause(e));
1320 webRTCWrapper.close();
1321 sendSessionTerminate(
1322 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1323 return;
1324 }
1325 processCandidates(contentMap.contents.entrySet());
1326 }
1327
1328 private void sendSessionAccept() {
1329 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1330 if (rtpContentMap == null) {
1331 throw new IllegalStateException("initiator RTP Content Map has not been set");
1332 }
1333 final SessionDescription offer;
1334 try {
1335 offer = SessionDescription.of(rtpContentMap, true);
1336 } catch (final IllegalArgumentException | NullPointerException e) {
1337 Log.d(
1338 Config.LOGTAG,
1339 id.account.getJid().asBareJid()
1340 + ": unable convert offer from session-initiate to SDP",
1341 e);
1342 webRTCWrapper.close();
1343 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1344 return;
1345 }
1346 sendSessionAccept(rtpContentMap.getMedia(), offer);
1347 }
1348
1349 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1350 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1351 }
1352
1353 private synchronized void sendSessionAccept(
1354 final Set<Media> media,
1355 final SessionDescription offer,
1356 final List<PeerConnection.IceServer> iceServers) {
1357 if (isTerminated()) {
1358 Log.w(
1359 Config.LOGTAG,
1360 id.account.getJid().asBareJid()
1361 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1362 return;
1363 }
1364 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1365 try {
1366 setupWebRTC(media, iceServers, !includeCandidates);
1367 } catch (final WebRTCWrapper.InitializationException e) {
1368 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1369 webRTCWrapper.close();
1370 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1371 return;
1372 }
1373 final org.webrtc.SessionDescription sdp =
1374 new org.webrtc.SessionDescription(
1375 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1376 try {
1377 this.webRTCWrapper.setRemoteDescription(sdp).get();
1378 addIceCandidatesFromBlackLog();
1379 org.webrtc.SessionDescription webRTCSessionDescription =
1380 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1381 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1382 } catch (final Exception e) {
1383 failureToAcceptSession(e);
1384 }
1385 }
1386
1387 private void failureToAcceptSession(final Throwable throwable) {
1388 if (isTerminated()) {
1389 return;
1390 }
1391 final Throwable rootCause = Throwables.getRootCause(throwable);
1392 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1393 webRTCWrapper.close();
1394 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1395 }
1396
1397 private void failureToPerformAction(
1398 final JinglePacket.Action action, final Throwable throwable) {
1399 if (isTerminated()) {
1400 return;
1401 }
1402 final Throwable rootCause = Throwables.getRootCause(throwable);
1403 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1404 webRTCWrapper.close();
1405 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1406 }
1407
1408 private void addIceCandidatesFromBlackLog() {
1409 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1410 while ((foo = this.pendingIceCandidates.poll()) != null) {
1411 processCandidate(foo);
1412 Log.d(
1413 Config.LOGTAG,
1414 id.account.getJid().asBareJid() + ": added candidate from back log");
1415 }
1416 }
1417
1418 private void prepareSessionAccept(
1419 final org.webrtc.SessionDescription webRTCSessionDescription,
1420 final boolean includeCandidates) {
1421 final SessionDescription sessionDescription =
1422 SessionDescription.parse(webRTCSessionDescription.description);
1423 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1424 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1425 if (includeCandidates) {
1426 candidates = parseCandidates(sessionDescription);
1427 } else {
1428 candidates = ImmutableMultimap.of();
1429 }
1430 this.responderRtpContentMap = respondingRtpContentMap;
1431 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1432 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1433 prepareOutgoingContentMap(respondingRtpContentMap);
1434 Futures.addCallback(
1435 outgoingContentMapFuture,
1436 new FutureCallback<>() {
1437 @Override
1438 public void onSuccess(final RtpContentMap outgoingContentMap) {
1439 if (includeCandidates) {
1440 Log.d(
1441 Config.LOGTAG,
1442 "including "
1443 + candidates.size()
1444 + " candidates in session accept");
1445 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1446 } else {
1447 sendSessionAccept(outgoingContentMap);
1448 }
1449 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1450 }
1451
1452 @Override
1453 public void onFailure(@NonNull Throwable throwable) {
1454 failureToAcceptSession(throwable);
1455 }
1456 },
1457 MoreExecutors.directExecutor());
1458 }
1459
1460 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1461 if (isTerminated()) {
1462 Log.w(
1463 Config.LOGTAG,
1464 id.account.getJid().asBareJid()
1465 + ": preparing session accept was too slow. already terminated. nothing to do.");
1466 return;
1467 }
1468 transitionOrThrow(State.SESSION_ACCEPTED);
1469 final JinglePacket sessionAccept =
1470 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1471 send(sessionAccept);
1472 }
1473
1474 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1475 final RtpContentMap rtpContentMap) {
1476 if (this.omemoVerification.hasDeviceId()) {
1477 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1478 verifiedPayloadFuture =
1479 id.account
1480 .getAxolotlService()
1481 .encrypt(
1482 rtpContentMap,
1483 id.with,
1484 omemoVerification.getDeviceId());
1485 return Futures.transform(
1486 verifiedPayloadFuture,
1487 verifiedPayload -> {
1488 omemoVerification.setOrEnsureEqual(verifiedPayload);
1489 return verifiedPayload.getPayload();
1490 },
1491 MoreExecutors.directExecutor());
1492 } else {
1493 return Futures.immediateFuture(rtpContentMap);
1494 }
1495 }
1496
1497 synchronized void deliveryMessage(
1498 final Jid from,
1499 final Element message,
1500 final String serverMessageId,
1501 final long timestamp) {
1502 Log.d(
1503 Config.LOGTAG,
1504 id.account.getJid().asBareJid()
1505 + ": delivered message to JingleRtpConnection "
1506 + message);
1507 switch (message.getName()) {
1508 case "propose" -> receivePropose(
1509 from, Propose.upgrade(message), serverMessageId, timestamp);
1510 case "proceed" -> receiveProceed(
1511 from, Proceed.upgrade(message), serverMessageId, timestamp);
1512 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1513 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1514 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1515 }
1516 }
1517
1518 void deliverFailedProceed(final String message) {
1519 Log.d(
1520 Config.LOGTAG,
1521 id.account.getJid().asBareJid()
1522 + ": receive message error for proceed message ("
1523 + Strings.nullToEmpty(message)
1524 + ")");
1525 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1526 webRTCWrapper.close();
1527 Log.d(
1528 Config.LOGTAG,
1529 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1530 this.finish();
1531 }
1532 }
1533
1534 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1535 final boolean originatedFromMyself =
1536 from.asBareJid().equals(id.account.getJid().asBareJid());
1537 if (originatedFromMyself) {
1538 if (transition(State.ACCEPTED)) {
1539 acceptedOnOtherDevice(serverMsgId, timestamp);
1540 } else {
1541 Log.d(
1542 Config.LOGTAG,
1543 id.account.getJid().asBareJid()
1544 + ": unable to transition to accept because already in state="
1545 + this.state);
1546 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1547 }
1548 } else {
1549 Log.d(
1550 Config.LOGTAG,
1551 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1552 }
1553 }
1554
1555 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1556 if (serverMsgId != null) {
1557 this.message.setServerMsgId(serverMsgId);
1558 }
1559 this.message.setTime(timestamp);
1560 this.message.setCarbon(true); // indicate that call was accepted on other device
1561 this.writeLogMessageSuccess(0);
1562 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1563 this.finish();
1564 }
1565
1566 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1567 final boolean originatedFromMyself =
1568 from.asBareJid().equals(id.account.getJid().asBareJid());
1569 // reject from another one of my clients
1570 if (originatedFromMyself) {
1571 receiveRejectFromMyself(serverMsgId, timestamp);
1572 } else if (isInitiator()) {
1573 if (from.equals(id.with)) {
1574 receiveRejectFromResponder();
1575 } else {
1576 Log.d(
1577 Config.LOGTAG,
1578 id.account.getJid()
1579 + ": ignoring reject from "
1580 + from
1581 + " for session with "
1582 + id.with);
1583 }
1584 } else {
1585 Log.d(
1586 Config.LOGTAG,
1587 id.account.getJid()
1588 + ": ignoring reject from "
1589 + from
1590 + " for session with "
1591 + id.with);
1592 }
1593 }
1594
1595 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1596 if (transition(State.REJECTED)) {
1597 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1598 this.finish();
1599 if (serverMsgId != null) {
1600 this.message.setServerMsgId(serverMsgId);
1601 }
1602 this.message.setTime(timestamp);
1603 this.message.setCarbon(true); // indicate that call was rejected on other device
1604 writeLogMessageMissed();
1605 } else {
1606 Log.d(
1607 Config.LOGTAG,
1608 "not able to transition into REJECTED because already in " + this.state);
1609 }
1610 }
1611
1612 private void receiveRejectFromResponder() {
1613 if (isInState(State.PROCEED)) {
1614 Log.d(
1615 Config.LOGTAG,
1616 id.account.getJid()
1617 + ": received reject while still in proceed. callee reconsidered");
1618 closeTransitionLogFinish(State.REJECTED_RACED);
1619 return;
1620 }
1621 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1622 Log.d(
1623 Config.LOGTAG,
1624 id.account.getJid()
1625 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1626 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1627 return;
1628 }
1629 Log.d(
1630 Config.LOGTAG,
1631 id.account.getJid()
1632 + ": ignoring reject from responder because already in state "
1633 + this.state);
1634 }
1635
1636 private void receivePropose(
1637 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1638 final boolean originatedFromMyself =
1639 from.asBareJid().equals(id.account.getJid().asBareJid());
1640 if (originatedFromMyself) {
1641 Log.d(
1642 Config.LOGTAG,
1643 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1644 } else if (transition(
1645 State.PROPOSED,
1646 () -> {
1647 final Collection<RtpDescription> descriptions =
1648 Collections2.transform(
1649 Collections2.filter(
1650 propose.getDescriptions(),
1651 d -> d instanceof RtpDescription),
1652 input -> (RtpDescription) input);
1653 final Collection<Media> media =
1654 Collections2.transform(descriptions, RtpDescription::getMedia);
1655 Preconditions.checkState(
1656 !media.contains(Media.UNKNOWN),
1657 "RTP descriptions contain unknown media");
1658 Log.d(
1659 Config.LOGTAG,
1660 id.account.getJid().asBareJid()
1661 + ": received session proposal from "
1662 + from
1663 + " for "
1664 + media);
1665 this.setProposedMedia(Sets.newHashSet(media));
1666 })) {
1667 if (serverMsgId != null) {
1668 this.message.setServerMsgId(serverMsgId);
1669 }
1670 this.message.setTime(timestamp);
1671 startRinging();
1672 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1673 sendJingleMessage("ringing");
1674 }
1675 } else {
1676 Log.d(
1677 Config.LOGTAG,
1678 id.account.getJid()
1679 + ": ignoring session proposal because already in "
1680 + state);
1681 }
1682 }
1683
1684 private void startRinging() {
1685 this.callIntegration.setRinging();
1686 Log.d(
1687 Config.LOGTAG,
1688 id.account.getJid().asBareJid()
1689 + ": received call from "
1690 + id.with
1691 + ". start ringing");
1692 ringingTimeoutFuture =
1693 jingleConnectionManager.schedule(
1694 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1695 if (CallIntegration.selfManaged(xmppConnectionService)) {
1696 return;
1697 }
1698 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1699 }
1700
1701 private synchronized void ringingTimeout() {
1702 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1703 switch (this.state) {
1704 case PROPOSED -> {
1705 message.markUnread();
1706 rejectCallFromProposed();
1707 }
1708 case SESSION_INITIALIZED -> {
1709 message.markUnread();
1710 rejectCallFromSessionInitiate();
1711 }
1712 }
1713 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1714 }
1715
1716 private void cancelRingingTimeout() {
1717 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1718 if (future != null && !future.isCancelled()) {
1719 future.cancel(false);
1720 }
1721 }
1722
1723 private void receiveProceed(
1724 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1725 final Set<Media> media =
1726 Preconditions.checkNotNull(
1727 this.proposedMedia, "Proposed media has to be set before handling proceed");
1728 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1729 if (from.equals(id.with)) {
1730 if (isInitiator()) {
1731 if (transition(State.PROCEED)) {
1732 if (serverMsgId != null) {
1733 this.message.setServerMsgId(serverMsgId);
1734 }
1735 this.message.setTime(timestamp);
1736 final Integer remoteDeviceId = proceed.getDeviceId();
1737 if (isOmemoEnabled()) {
1738 this.omemoVerification.setDeviceId(remoteDeviceId);
1739 } else {
1740 if (remoteDeviceId != null) {
1741 Log.d(
1742 Config.LOGTAG,
1743 id.account.getJid().asBareJid()
1744 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1745 }
1746 this.omemoVerification.setDeviceId(null);
1747 }
1748 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1749 } else {
1750 Log.d(
1751 Config.LOGTAG,
1752 String.format(
1753 "%s: ignoring proceed because already in %s",
1754 id.account.getJid().asBareJid(), this.state));
1755 }
1756 } else {
1757 Log.d(
1758 Config.LOGTAG,
1759 String.format(
1760 "%s: ignoring proceed because we were not initializing",
1761 id.account.getJid().asBareJid()));
1762 }
1763 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1764 if (transition(State.ACCEPTED)) {
1765 Log.d(
1766 Config.LOGTAG,
1767 id.account.getJid().asBareJid()
1768 + ": moved session with "
1769 + id.with
1770 + " into state accepted after received carbon copied proceed");
1771 acceptedOnOtherDevice(serverMsgId, timestamp);
1772 }
1773 } else {
1774 Log.d(
1775 Config.LOGTAG,
1776 String.format(
1777 "%s: ignoring proceed from %s. was expected from %s",
1778 id.account.getJid().asBareJid(), from, id.with));
1779 }
1780 }
1781
1782 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1783 if (from.equals(id.with)) {
1784 final State target =
1785 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1786 if (transition(target)) {
1787 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1788 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1789 Log.d(
1790 Config.LOGTAG,
1791 id.account.getJid().asBareJid()
1792 + ": session with "
1793 + id.with
1794 + " has been retracted (serverMsgId="
1795 + serverMsgId
1796 + ")");
1797 if (serverMsgId != null) {
1798 this.message.setServerMsgId(serverMsgId);
1799 }
1800 this.message.setTime(timestamp);
1801 if (target == State.RETRACTED) {
1802 this.message.markUnread();
1803 }
1804 writeLogMessageMissed();
1805 finish();
1806 } else {
1807 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1808 }
1809 } else {
1810 // TODO parse retract from self
1811 Log.d(
1812 Config.LOGTAG,
1813 id.account.getJid().asBareJid()
1814 + ": received retract from "
1815 + from
1816 + ". expected retract from"
1817 + id.with
1818 + ". ignoring");
1819 }
1820 }
1821
1822 public void sendSessionInitiate() {
1823 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1824 }
1825
1826 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1827 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1828 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1829 }
1830
1831 private synchronized void sendSessionInitiate(
1832 final Set<Media> media,
1833 final State targetState,
1834 final List<PeerConnection.IceServer> iceServers) {
1835 if (isTerminated()) {
1836 Log.w(
1837 Config.LOGTAG,
1838 id.account.getJid().asBareJid()
1839 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1840 return;
1841 }
1842 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1843 try {
1844 setupWebRTC(media, iceServers, !includeCandidates);
1845 } catch (final WebRTCWrapper.InitializationException e) {
1846 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1847 webRTCWrapper.close();
1848 sendRetract(Reason.ofThrowable(e));
1849 return;
1850 }
1851 try {
1852 org.webrtc.SessionDescription webRTCSessionDescription =
1853 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1854 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1855 } catch (final Exception e) {
1856 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1857 // exceptions
1858 failureToInitiateSession(e, targetState);
1859 }
1860 }
1861
1862 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1863 if (isTerminated()) {
1864 return;
1865 }
1866 Log.d(
1867 Config.LOGTAG,
1868 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1869 Throwables.getRootCause(throwable));
1870 webRTCWrapper.close();
1871 final Reason reason = Reason.ofThrowable(throwable);
1872 if (isInState(targetState)) {
1873 sendSessionTerminate(reason, throwable.getMessage());
1874 } else {
1875 sendRetract(reason);
1876 }
1877 }
1878
1879 private void sendRetract(final Reason reason) {
1880 // TODO embed reason into retract
1881 sendJingleMessage("retract", id.with.asBareJid());
1882 transitionOrThrow(reasonToState(reason));
1883 this.finish();
1884 }
1885
1886 private void prepareSessionInitiate(
1887 final org.webrtc.SessionDescription webRTCSessionDescription,
1888 final boolean includeCandidates,
1889 final State targetState) {
1890 final SessionDescription sessionDescription =
1891 SessionDescription.parse(webRTCSessionDescription.description);
1892 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1893 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1894 if (includeCandidates) {
1895 candidates = parseCandidates(sessionDescription);
1896 } else {
1897 candidates = ImmutableMultimap.of();
1898 }
1899 this.initiatorRtpContentMap = rtpContentMap;
1900 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1901 encryptSessionInitiate(rtpContentMap);
1902 Futures.addCallback(
1903 outgoingContentMapFuture,
1904 new FutureCallback<>() {
1905 @Override
1906 public void onSuccess(final RtpContentMap outgoingContentMap) {
1907 if (includeCandidates) {
1908 Log.d(
1909 Config.LOGTAG,
1910 "including "
1911 + candidates.size()
1912 + " candidates in session initiate");
1913 sendSessionInitiate(
1914 outgoingContentMap.withCandidates(candidates), targetState);
1915 } else {
1916 sendSessionInitiate(outgoingContentMap, targetState);
1917 }
1918 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1919 }
1920
1921 @Override
1922 public void onFailure(@NonNull final Throwable throwable) {
1923 failureToInitiateSession(throwable, targetState);
1924 }
1925 },
1926 MoreExecutors.directExecutor());
1927 }
1928
1929 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1930 if (isTerminated()) {
1931 Log.w(
1932 Config.LOGTAG,
1933 id.account.getJid().asBareJid()
1934 + ": preparing session was too slow. already terminated. nothing to do.");
1935 return;
1936 }
1937 this.transitionOrThrow(targetState);
1938 final JinglePacket sessionInitiate =
1939 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1940 send(sessionInitiate);
1941 }
1942
1943 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1944 final RtpContentMap rtpContentMap) {
1945 if (this.omemoVerification.hasDeviceId()) {
1946 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1947 verifiedPayloadFuture =
1948 id.account
1949 .getAxolotlService()
1950 .encrypt(
1951 rtpContentMap,
1952 id.with,
1953 omemoVerification.getDeviceId());
1954 final ListenableFuture<RtpContentMap> future =
1955 Futures.transform(
1956 verifiedPayloadFuture,
1957 verifiedPayload -> {
1958 omemoVerification.setSessionFingerprint(
1959 verifiedPayload.getFingerprint());
1960 return verifiedPayload.getPayload();
1961 },
1962 MoreExecutors.directExecutor());
1963 if (Config.REQUIRE_RTP_VERIFICATION) {
1964 return future;
1965 }
1966 return Futures.catching(
1967 future,
1968 CryptoFailedException.class,
1969 e -> {
1970 Log.w(
1971 Config.LOGTAG,
1972 id.account.getJid().asBareJid()
1973 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1974 e);
1975 return rtpContentMap;
1976 },
1977 MoreExecutors.directExecutor());
1978 } else {
1979 return Futures.immediateFuture(rtpContentMap);
1980 }
1981 }
1982
1983 protected void sendSessionTerminate(final Reason reason) {
1984 sendSessionTerminate(reason, null);
1985 }
1986
1987 protected void sendSessionTerminate(final Reason reason, final String text) {
1988 sendSessionTerminate(reason, text, this::writeLogMessage);
1989 sendJingleMessageFinish(reason);
1990 }
1991
1992 private void sendTransportInfo(
1993 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1994 final RtpContentMap transportInfo;
1995 try {
1996 final RtpContentMap rtpContentMap =
1997 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1998 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1999 } catch (final Exception e) {
2000 Log.d(
2001 Config.LOGTAG,
2002 id.account.getJid().asBareJid()
2003 + ": unable to prepare transport-info from candidate for content="
2004 + contentName);
2005 return;
2006 }
2007 final JinglePacket jinglePacket =
2008 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2009 send(jinglePacket);
2010 }
2011
2012 public RtpEndUserState getEndUserState() {
2013 switch (this.state) {
2014 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2015 if (isInitiator()) {
2016 return RtpEndUserState.RINGING;
2017 } else {
2018 return RtpEndUserState.INCOMING_CALL;
2019 }
2020 }
2021 case PROCEED -> {
2022 if (isInitiator()) {
2023 return RtpEndUserState.RINGING;
2024 } else {
2025 return RtpEndUserState.ACCEPTING_CALL;
2026 }
2027 }
2028 case SESSION_INITIALIZED_PRE_APPROVED -> {
2029 if (isInitiator()) {
2030 return RtpEndUserState.RINGING;
2031 } else {
2032 return RtpEndUserState.CONNECTING;
2033 }
2034 }
2035 case SESSION_ACCEPTED -> {
2036 final ContentAddition ca = getPendingContentAddition();
2037 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2038 return RtpEndUserState.INCOMING_CONTENT_ADD;
2039 }
2040 return getPeerConnectionStateAsEndUserState();
2041 }
2042 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2043 if (isInitiator()) {
2044 return RtpEndUserState.DECLINED_OR_BUSY;
2045 } else {
2046 return RtpEndUserState.ENDED;
2047 }
2048 }
2049 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2050 return RtpEndUserState.ENDED;
2051 }
2052 case RETRACTED_RACED -> {
2053 if (isInitiator()) {
2054 return RtpEndUserState.ENDED;
2055 } else {
2056 return RtpEndUserState.RETRACTED;
2057 }
2058 }
2059 case TERMINATED_CONNECTIVITY_ERROR -> {
2060 return zeroDuration()
2061 ? RtpEndUserState.CONNECTIVITY_ERROR
2062 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2063 }
2064 case TERMINATED_APPLICATION_FAILURE -> {
2065 return RtpEndUserState.APPLICATION_ERROR;
2066 }
2067 case TERMINATED_SECURITY_ERROR -> {
2068 return RtpEndUserState.SECURITY_ERROR;
2069 }
2070 }
2071 throw new IllegalStateException(
2072 String.format("%s has no equivalent EndUserState", this.state));
2073 }
2074
2075 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2076 final PeerConnection.PeerConnectionState state;
2077 try {
2078 state = webRTCWrapper.getState();
2079 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2080 // We usually close the WebRTCWrapper *before* transitioning so we might still
2081 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2082 return RtpEndUserState.ENDING_CALL;
2083 }
2084 return switch (state) {
2085 case CONNECTED -> RtpEndUserState.CONNECTED;
2086 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2087 case CLOSED -> RtpEndUserState.ENDING_CALL;
2088 default -> zeroDuration()
2089 ? RtpEndUserState.CONNECTIVITY_ERROR
2090 : RtpEndUserState.RECONNECTING;
2091 };
2092 }
2093
2094 private boolean isPeerConnectionConnected() {
2095 try {
2096 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2097 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2098 return false;
2099 }
2100 }
2101
2102 private void updateCallIntegrationState() {
2103 switch (this.state) {
2104 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2105 if (isInitiator()) {
2106 this.callIntegration.setDialing();
2107 } else {
2108 this.callIntegration.setRinging();
2109 }
2110 }
2111 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2112 if (isInitiator()) {
2113 this.callIntegration.setDialing();
2114 } else {
2115 this.callIntegration.setInitialized();
2116 }
2117 }
2118 case SESSION_ACCEPTED -> {
2119 if (isPeerConnectionConnected()) {
2120 this.callIntegration.setActive();
2121 } else {
2122 this.callIntegration.setInitialized();
2123 }
2124 }
2125 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2126 if (isInitiator()) {
2127 this.callIntegration.busy();
2128 } else {
2129 this.callIntegration.rejected();
2130 }
2131 }
2132 case TERMINATED_SUCCESS -> this.callIntegration.success();
2133 case ACCEPTED -> this.callIntegration.accepted();
2134 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2135 .retracted();
2136 case TERMINATED_CONNECTIVITY_ERROR,
2137 TERMINATED_APPLICATION_FAILURE,
2138 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2139 default -> throw new IllegalStateException(
2140 String.format("%s is not handled", this.state));
2141 }
2142 }
2143
2144 public ContentAddition getPendingContentAddition() {
2145 final RtpContentMap in = this.incomingContentAdd;
2146 final RtpContentMap out = this.outgoingContentAdd;
2147 if (out != null) {
2148 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2149 } else if (in != null) {
2150 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2151 } else {
2152 return null;
2153 }
2154 }
2155
2156 public Set<Media> getMedia() {
2157 final State current = getState();
2158 if (current == State.NULL) {
2159 if (isInitiator()) {
2160 return Preconditions.checkNotNull(
2161 this.proposedMedia, "RTP connection has not been initialized properly");
2162 }
2163 throw new IllegalStateException("RTP connection has not been initialized yet");
2164 }
2165 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2166 return Preconditions.checkNotNull(
2167 this.proposedMedia, "RTP connection has not been initialized properly");
2168 }
2169 final RtpContentMap localContentMap = getLocalContentMap();
2170 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2171 if (localContentMap != null) {
2172 return localContentMap.getMedia();
2173 } else if (initiatorContentMap != null) {
2174 return initiatorContentMap.getMedia();
2175 } else if (isTerminated()) {
2176 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2177 } else {
2178 return Preconditions.checkNotNull(
2179 this.proposedMedia, "RTP connection has not been initialized properly");
2180 }
2181 }
2182
2183 public boolean isVerified() {
2184 final String fingerprint = this.omemoVerification.getFingerprint();
2185 if (fingerprint == null) {
2186 return false;
2187 }
2188 final FingerprintStatus status =
2189 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2190 return status != null && status.isVerified();
2191 }
2192
2193 public boolean addMedia(final Media media) {
2194 final Set<Media> currentMedia = getMedia();
2195 if (currentMedia.contains(media)) {
2196 throw new IllegalStateException(String.format("%s has already been proposed", media));
2197 }
2198 // TODO add state protection - can only add while ACCEPTED or so
2199 Log.d(Config.LOGTAG, "adding media: " + media);
2200 return webRTCWrapper.addTrack(media);
2201 }
2202
2203 public synchronized void acceptCall() {
2204 switch (this.state) {
2205 case PROPOSED -> {
2206 cancelRingingTimeout();
2207 acceptCallFromProposed();
2208 }
2209 case SESSION_INITIALIZED -> {
2210 cancelRingingTimeout();
2211 acceptCallFromSessionInitialized();
2212 }
2213 case ACCEPTED -> Log.w(
2214 Config.LOGTAG,
2215 id.account.getJid().asBareJid()
2216 + ": the call has already been accepted with another client. UI was just lagging behind");
2217 case PROCEED, SESSION_ACCEPTED -> Log.w(
2218 Config.LOGTAG,
2219 id.account.getJid().asBareJid()
2220 + ": the call has already been accepted. user probably double tapped the UI");
2221 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2222 }
2223 }
2224
2225 public synchronized void rejectCall() {
2226 if (isTerminated()) {
2227 Log.w(
2228 Config.LOGTAG,
2229 id.account.getJid().asBareJid()
2230 + ": received rejectCall() when session has already been terminated. nothing to do");
2231 return;
2232 }
2233 switch (this.state) {
2234 case PROPOSED -> rejectCallFromProposed();
2235 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2236 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2237 }
2238 }
2239
2240 public synchronized void integrationFailure() {
2241 final var state = getState();
2242 if (state == State.PROPOSED) {
2243 Log.e(
2244 Config.LOGTAG,
2245 id.account.getJid().asBareJid()
2246 + ": failed call integration in state proposed");
2247 rejectCallFromProposed();
2248 } else if (state == State.SESSION_INITIALIZED) {
2249 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2250 this.webRTCWrapper.close();
2251 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2252 } else {
2253 throw new IllegalStateException(
2254 String.format("Can not fail integration in state %s", state));
2255 }
2256 }
2257
2258 public synchronized void endCall() {
2259 if (isTerminated()) {
2260 Log.w(
2261 Config.LOGTAG,
2262 id.account.getJid().asBareJid()
2263 + ": received endCall() when session has already been terminated. nothing to do");
2264 return;
2265 }
2266 if (isInState(State.PROPOSED) && isResponder()) {
2267 rejectCallFromProposed();
2268 return;
2269 }
2270 if (isInState(State.PROCEED)) {
2271 if (isInitiator()) {
2272 retractFromProceed();
2273 } else {
2274 rejectCallFromProceed();
2275 }
2276 return;
2277 }
2278 if (isInitiator()
2279 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2280 this.webRTCWrapper.close();
2281 sendSessionTerminate(Reason.CANCEL);
2282 return;
2283 }
2284 if (isInState(State.SESSION_INITIALIZED)) {
2285 rejectCallFromSessionInitiate();
2286 return;
2287 }
2288 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2289 this.webRTCWrapper.close();
2290 sendSessionTerminate(Reason.SUCCESS);
2291 return;
2292 }
2293 if (isInState(
2294 State.TERMINATED_APPLICATION_FAILURE,
2295 State.TERMINATED_CONNECTIVITY_ERROR,
2296 State.TERMINATED_DECLINED_OR_BUSY)) {
2297 Log.d(
2298 Config.LOGTAG,
2299 "ignoring request to end call because already in state " + this.state);
2300 return;
2301 }
2302 throw new IllegalStateException(
2303 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2304 }
2305
2306 private void retractFromProceed() {
2307 Log.d(Config.LOGTAG, "retract from proceed");
2308 this.sendJingleMessage("retract");
2309 closeTransitionLogFinish(State.RETRACTED_RACED);
2310 }
2311
2312 private void closeTransitionLogFinish(final State state) {
2313 this.webRTCWrapper.close();
2314 transitionOrThrow(state);
2315 writeLogMessage(state);
2316 finish();
2317 }
2318
2319 private void setupWebRTC(
2320 final Set<Media> media,
2321 final List<PeerConnection.IceServer> iceServers,
2322 final boolean trickle)
2323 throws WebRTCWrapper.InitializationException {
2324 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2325 this.webRTCWrapper.setup(this.xmppConnectionService);
2326 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2327 }
2328
2329 private void acceptCallFromProposed() {
2330 transitionOrThrow(State.PROCEED);
2331 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2332 this.callIntegration.startAudioRouting();
2333 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2334 this.sendJingleMessage("proceed");
2335 }
2336
2337 private void rejectCallFromProposed() {
2338 transitionOrThrow(State.REJECTED);
2339 writeLogMessageMissed();
2340 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2341 this.sendJingleMessage("reject");
2342 finish();
2343 }
2344
2345 private void rejectCallFromProceed() {
2346 this.sendJingleMessage("reject");
2347 closeTransitionLogFinish(State.REJECTED_RACED);
2348 }
2349
2350 private void rejectCallFromSessionInitiate() {
2351 webRTCWrapper.close();
2352 sendSessionTerminate(Reason.DECLINE);
2353 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2354 }
2355
2356 private void sendJingleMessage(final String action) {
2357 sendJingleMessage(action, id.with);
2358 }
2359
2360 private void sendJingleMessage(final String action, final Jid to) {
2361 final MessagePacket messagePacket = new MessagePacket();
2362 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2363 messagePacket.setTo(to);
2364 final Element intent =
2365 messagePacket
2366 .addChild(action, Namespace.JINGLE_MESSAGE)
2367 .setAttribute("id", id.sessionId);
2368 if ("proceed".equals(action)) {
2369 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2370 if (isOmemoEnabled()) {
2371 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2372 final Element device =
2373 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2374 device.setAttribute("id", deviceId);
2375 }
2376 }
2377 messagePacket.addChild("store", "urn:xmpp:hints");
2378 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2379 }
2380
2381 private void sendJingleMessageFinish(final Reason reason) {
2382 final var account = id.getAccount();
2383 final MessagePacket messagePacket =
2384 xmppConnectionService
2385 .getMessageGenerator()
2386 .sessionFinish(id.with, id.sessionId, reason);
2387 xmppConnectionService.sendMessagePacket(account, messagePacket);
2388 }
2389
2390 private boolean isOmemoEnabled() {
2391 final Conversational conversational = message.getConversation();
2392 if (conversational instanceof Conversation) {
2393 return ((Conversation) conversational).getNextEncryption()
2394 == Message.ENCRYPTION_AXOLOTL;
2395 }
2396 return false;
2397 }
2398
2399 private void acceptCallFromSessionInitialized() {
2400 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2401 this.callIntegration.startAudioRouting();
2402 sendSessionAccept();
2403 }
2404
2405 @Override
2406 protected synchronized boolean transition(final State target, final Runnable runnable) {
2407 if (super.transition(target, runnable)) {
2408 updateEndUserState();
2409 updateOngoingCallNotification();
2410 return true;
2411 } else {
2412 return false;
2413 }
2414 }
2415
2416 @Override
2417 public void onIceCandidate(final IceCandidate iceCandidate) {
2418 final RtpContentMap rtpContentMap =
2419 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2420 final IceUdpTransportInfo.Credentials credentials;
2421 try {
2422 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2423 } catch (final IllegalArgumentException e) {
2424 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2425 return;
2426 }
2427 final String uFrag = credentials.ufrag;
2428 final IceUdpTransportInfo.Candidate candidate =
2429 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2430 if (candidate == null) {
2431 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2432 return;
2433 }
2434 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2435 sendTransportInfo(iceCandidate.sdpMid, candidate);
2436 }
2437
2438 @Override
2439 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2440 Log.d(
2441 Config.LOGTAG,
2442 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2443 this.stateHistory.add(newState);
2444 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2445 this.sessionDuration.start();
2446 updateOngoingCallNotification();
2447 } else if (this.sessionDuration.isRunning()) {
2448 this.sessionDuration.stop();
2449 updateOngoingCallNotification();
2450 }
2451
2452 final boolean neverConnected =
2453 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2454
2455 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2456 if (neverConnected) {
2457 if (isTerminated()) {
2458 Log.d(
2459 Config.LOGTAG,
2460 id.account.getJid().asBareJid()
2461 + ": not sending session-terminate after connectivity error because session is already in state "
2462 + this.state);
2463 return;
2464 }
2465 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2466 return;
2467 } else {
2468 this.restartIce();
2469 }
2470 }
2471 updateEndUserState();
2472 }
2473
2474 private void restartIce() {
2475 this.stateHistory.clear();
2476 this.webRTCWrapper.restartIceAsync();
2477 }
2478
2479 @Override
2480 public void onRenegotiationNeeded() {
2481 this.webRTCWrapper.execute(this::renegotiate);
2482 }
2483
2484 private void renegotiate() {
2485 final SessionDescription sessionDescription;
2486 try {
2487 sessionDescription = setLocalSessionDescription();
2488 } catch (final Exception e) {
2489 final Throwable cause = Throwables.getRootCause(e);
2490 webRTCWrapper.close();
2491 if (isTerminated()) {
2492 Log.d(
2493 Config.LOGTAG,
2494 "failed to renegotiate. session was already terminated",
2495 cause);
2496 return;
2497 }
2498 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2499 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2500 return;
2501 }
2502 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2503 final RtpContentMap currentContentMap = getLocalContentMap();
2504 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2505 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2506
2507 Log.d(
2508 Config.LOGTAG,
2509 id.getAccount().getJid().asBareJid()
2510 + ": renegotiate. iceRestart="
2511 + iceRestart
2512 + " content id diff="
2513 + diff);
2514
2515 if (diff.hasModifications() && iceRestart) {
2516 webRTCWrapper.close();
2517 sendSessionTerminate(
2518 Reason.FAILED_APPLICATION,
2519 "WebRTC unexpectedly tried to modify content and transport at once");
2520 return;
2521 }
2522
2523 if (iceRestart) {
2524 initiateIceRestart(rtpContentMap);
2525 return;
2526 } else if (diff.isEmpty()) {
2527 Log.d(
2528 Config.LOGTAG,
2529 "renegotiation. nothing to do. SignalingState="
2530 + this.webRTCWrapper.getSignalingState());
2531 }
2532
2533 if (diff.added.size() > 0) {
2534 modifyLocalContentMap(rtpContentMap);
2535 sendContentAdd(rtpContentMap, diff.added);
2536 }
2537 }
2538
2539 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2540 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2541 final JinglePacket jinglePacket =
2542 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2543 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2544 jinglePacket.setTo(id.with);
2545 xmppConnectionService.sendIqPacket(
2546 id.account,
2547 jinglePacket,
2548 (account, response) -> {
2549 if (response.getType() == IqPacket.TYPE.RESULT) {
2550 Log.d(Config.LOGTAG, "received success to our ice restart");
2551 setLocalContentMap(rtpContentMap);
2552 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2553 return;
2554 }
2555 if (response.getType() == IqPacket.TYPE.ERROR) {
2556 if (isTieBreak(response)) {
2557 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2558 return;
2559 }
2560 handleIqErrorResponse(response);
2561 }
2562 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2563 handleIqTimeoutResponse(response);
2564 }
2565 });
2566 }
2567
2568 private boolean isTieBreak(final IqPacket response) {
2569 final Element error = response.findChild("error");
2570 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2571 }
2572
2573 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2574 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2575 this.outgoingContentAdd = contentAdd;
2576 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2577 prepareOutgoingContentMap(contentAdd);
2578 Futures.addCallback(
2579 outgoingContentMapFuture,
2580 new FutureCallback<>() {
2581 @Override
2582 public void onSuccess(final RtpContentMap outgoingContentMap) {
2583 sendContentAdd(outgoingContentMap);
2584 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2585 }
2586
2587 @Override
2588 public void onFailure(@NonNull Throwable throwable) {
2589 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2590 }
2591 },
2592 MoreExecutors.directExecutor());
2593 }
2594
2595 private void sendContentAdd(final RtpContentMap contentAdd) {
2596
2597 final JinglePacket jinglePacket =
2598 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2599 jinglePacket.setTo(id.with);
2600 xmppConnectionService.sendIqPacket(
2601 id.account,
2602 jinglePacket,
2603 (connection, response) -> {
2604 if (response.getType() == IqPacket.TYPE.RESULT) {
2605 Log.d(
2606 Config.LOGTAG,
2607 id.getAccount().getJid().asBareJid()
2608 + ": received ACK to our content-add");
2609 return;
2610 }
2611 if (response.getType() == IqPacket.TYPE.ERROR) {
2612 if (isTieBreak(response)) {
2613 this.outgoingContentAdd = null;
2614 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2615 return;
2616 }
2617 handleIqErrorResponse(response);
2618 }
2619 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2620 handleIqTimeoutResponse(response);
2621 }
2622 });
2623 }
2624
2625 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2626 if (isInitiator()) {
2627 this.initiatorRtpContentMap = rtpContentMap;
2628 } else {
2629 this.responderRtpContentMap = rtpContentMap;
2630 }
2631 }
2632
2633 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2634 if (isInitiator()) {
2635 this.responderRtpContentMap = rtpContentMap;
2636 } else {
2637 this.initiatorRtpContentMap = rtpContentMap;
2638 }
2639 }
2640
2641 // this method is to be used for content map modifications that modify media
2642 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2643 final RtpContentMap activeContents = rtpContentMap.activeContents();
2644 setLocalContentMap(activeContents);
2645 this.callIntegration.setAudioDeviceWhenAvailable(
2646 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2647 updateEndUserState();
2648 }
2649
2650 private SessionDescription setLocalSessionDescription()
2651 throws ExecutionException, InterruptedException {
2652 final org.webrtc.SessionDescription sessionDescription =
2653 this.webRTCWrapper.setLocalDescription(false).get();
2654 return SessionDescription.parse(sessionDescription.description);
2655 }
2656
2657 private void closeWebRTCSessionAfterFailedConnection() {
2658 this.webRTCWrapper.close();
2659 synchronized (this) {
2660 if (isTerminated()) {
2661 Log.d(
2662 Config.LOGTAG,
2663 id.account.getJid().asBareJid()
2664 + ": no need to send session-terminate after failed connection. Other party already did");
2665 return;
2666 }
2667 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2668 }
2669 }
2670
2671 public boolean zeroDuration() {
2672 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2673 }
2674
2675 public long getCallDuration() {
2676 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2677 }
2678
2679 @Override
2680 public CallIntegration getCallIntegration() {
2681 return this.callIntegration;
2682 }
2683
2684 public boolean isMicrophoneEnabled() {
2685 return webRTCWrapper.isMicrophoneEnabled();
2686 }
2687
2688 public boolean setMicrophoneEnabled(final boolean enabled) {
2689 return webRTCWrapper.setMicrophoneEnabled(enabled);
2690 }
2691
2692 public boolean isVideoEnabled() {
2693 return webRTCWrapper.isVideoEnabled();
2694 }
2695
2696 public void setVideoEnabled(final boolean enabled) {
2697 webRTCWrapper.setVideoEnabled(enabled);
2698 }
2699
2700 public boolean isCameraSwitchable() {
2701 return webRTCWrapper.isCameraSwitchable();
2702 }
2703
2704 public boolean isFrontCamera() {
2705 return webRTCWrapper.isFrontCamera();
2706 }
2707
2708 public ListenableFuture<Boolean> switchCamera() {
2709 return webRTCWrapper.switchCamera();
2710 }
2711
2712 @Override
2713 public synchronized void onCallIntegrationShowIncomingCallUi() {
2714 if (isTerminated()) {
2715 // there might be race conditions with the call integration service invoking this
2716 // callback when the rtp session has already ended.
2717 Log.w(
2718 Config.LOGTAG,
2719 "CallIntegration requested incoming call UI but session was already terminated");
2720 return;
2721 }
2722 // TODO apparently this can be called too early as well?
2723 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2724 }
2725
2726 @Override
2727 public void onCallIntegrationDisconnect() {
2728 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2729 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2730 rejectCall();
2731 } else {
2732 endCall();
2733 }
2734 }
2735
2736 @Override
2737 public void onCallIntegrationReject() {
2738 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2739 try {
2740 rejectCall();
2741 } catch (final IllegalStateException e) {
2742 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2743 }
2744 }
2745
2746 @Override
2747 public void onCallIntegrationAnswer() {
2748 // we need to start the UI to a) show it and b) be able to ask for permissions
2749 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2750 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2751 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2752 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2753 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2754 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2755 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2756 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2757 xmppConnectionService.startActivity(intent);
2758 }
2759
2760 @Override
2761 public void onCallIntegrationSilence() {
2762 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2763 }
2764
2765 @Override
2766 public void onAudioDeviceChanged(
2767 final CallIntegration.AudioDevice selectedAudioDevice,
2768 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2769 Log.d(
2770 Config.LOGTAG,
2771 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2772 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2773 selectedAudioDevice, availableAudioDevices);
2774 }
2775
2776 private void updateEndUserState() {
2777 final RtpEndUserState endUserState = getEndUserState();
2778 this.updateCallIntegrationState();
2779 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2780 id.account, id.with, id.sessionId, endUserState);
2781 }
2782
2783 private void updateOngoingCallNotification() {
2784 final State state = this.state;
2785 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2786 final boolean reconnecting;
2787 if (state == State.SESSION_ACCEPTED) {
2788 reconnecting =
2789 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2790 } else {
2791 reconnecting = false;
2792 }
2793 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2794 } else {
2795 xmppConnectionService.removeOngoingCall();
2796 }
2797 }
2798
2799 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2800 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2801 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2802 request.setTo(id.account.getDomain());
2803 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2804 xmppConnectionService.sendIqPacket(
2805 id.account,
2806 request,
2807 (account, response) -> {
2808 final var iceServers = IceServers.parse(response);
2809 if (iceServers.isEmpty()) {
2810 Log.w(
2811 Config.LOGTAG,
2812 id.account.getJid().asBareJid()
2813 + ": no ICE server found "
2814 + response);
2815 }
2816 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2817 });
2818 } else {
2819 Log.w(
2820 Config.LOGTAG,
2821 id.account.getJid().asBareJid() + ": has no external service discovery");
2822 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2823 }
2824 }
2825
2826 @Override
2827 protected void terminateTransport() {
2828 this.webRTCWrapper.close();
2829 }
2830
2831 @Override
2832 protected void finish() {
2833 if (isTerminated()) {
2834 this.cancelRingingTimeout();
2835 this.callIntegration.verifyDisconnected();
2836 this.webRTCWrapper.verifyClosed();
2837 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2838 super.finish();
2839 } else {
2840 throw new IllegalStateException(
2841 String.format("Unable to call finish from %s", this.state));
2842 }
2843 }
2844
2845 private void writeLogMessage(final State state) {
2846 final long duration = getCallDuration();
2847 if (state == State.TERMINATED_SUCCESS
2848 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2849 writeLogMessageSuccess(duration);
2850 } else {
2851 writeLogMessageMissed();
2852 }
2853 }
2854
2855 private void writeLogMessageSuccess(final long duration) {
2856 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2857 this.writeMessage();
2858 }
2859
2860 private void writeLogMessageMissed() {
2861 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2862 this.writeMessage();
2863 }
2864
2865 private void writeMessage() {
2866 final Conversational conversational = message.getConversation();
2867 if (conversational instanceof Conversation) {
2868 ((Conversation) conversational).add(this.message);
2869 xmppConnectionService.createMessageAsync(message);
2870 xmppConnectionService.updateConversationUi();
2871 } else {
2872 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2873 }
2874 }
2875
2876 public Optional<VideoTrack> getLocalVideoTrack() {
2877 return webRTCWrapper.getLocalVideoTrack();
2878 }
2879
2880 public Optional<VideoTrack> getRemoteVideoTrack() {
2881 return webRTCWrapper.getRemoteVideoTrack();
2882 }
2883
2884 public EglBase.Context getEglBaseContext() {
2885 return webRTCWrapper.getEglBaseContext();
2886 }
2887
2888 void setProposedMedia(final Set<Media> media) {
2889 this.proposedMedia = media;
2890 this.callIntegration.setVideoState(
2891 Media.audioOnly(media)
2892 ? VideoProfile.STATE_AUDIO_ONLY
2893 : VideoProfile.STATE_BIDIRECTIONAL);
2894 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2895 }
2896
2897 public void fireStateUpdate() {
2898 final RtpEndUserState endUserState = getEndUserState();
2899 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2900 id.account, id.with, id.sessionId, endUserState);
2901 }
2902
2903 public boolean isSwitchToVideoAvailable() {
2904 final boolean prerequisite =
2905 Media.audioOnly(getMedia())
2906 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2907 .contains(getEndUserState());
2908 return prerequisite && remoteHasVideoFeature();
2909 }
2910
2911 private boolean remoteHasVideoFeature() {
2912 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2913 }
2914
2915 private boolean remoteHasSdpOfferAnswer() {
2916 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2917 }
2918
2919 @Override
2920 public Account getAccount() {
2921 return id.account;
2922 }
2923
2924 @Override
2925 public Jid getWith() {
2926 return id.with;
2927 }
2928
2929 @Override
2930 public String getSessionId() {
2931 return id.sessionId;
2932 }
2933
2934 private interface OnIceServersDiscovered {
2935 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2936 }
2937}