1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import androidx.annotation.NonNull;
8import androidx.annotation.Nullable;
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableMultimap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Iterables;
19import com.google.common.collect.Maps;
20import com.google.common.collect.Sets;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25import eu.siacs.conversations.BuildConfig;
26import eu.siacs.conversations.Config;
27import eu.siacs.conversations.crypto.axolotl.AxolotlService;
28import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
29import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
30import eu.siacs.conversations.entities.Account;
31import eu.siacs.conversations.entities.Conversation;
32import eu.siacs.conversations.entities.Conversational;
33import eu.siacs.conversations.entities.Message;
34import eu.siacs.conversations.entities.RtpSessionStatus;
35import eu.siacs.conversations.services.CallIntegration;
36import eu.siacs.conversations.ui.RtpSessionActivity;
37import eu.siacs.conversations.xml.Element;
38import eu.siacs.conversations.xml.Namespace;
39import eu.siacs.conversations.xmpp.Jid;
40import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
41import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
42import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
46import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
47import im.conversations.android.xmpp.model.jingle.Jingle;
48import im.conversations.android.xmpp.model.stanza.Iq;
49import java.util.Arrays;
50import java.util.Collection;
51import java.util.Collections;
52import java.util.LinkedList;
53import java.util.List;
54import java.util.Map;
55import java.util.Queue;
56import java.util.Set;
57import java.util.concurrent.ExecutionException;
58import java.util.concurrent.ScheduledFuture;
59import java.util.concurrent.TimeUnit;
60import org.webrtc.EglBase;
61import org.webrtc.IceCandidate;
62import org.webrtc.PeerConnection;
63import org.webrtc.VideoTrack;
64
65public class JingleRtpConnection extends AbstractJingleConnection
66 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
67
68 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
69 public static final List<State> STATES_SHOWING_ONGOING_CALL =
70 Arrays.asList(
71 State.PROPOSED,
72 State.PROCEED,
73 State.SESSION_INITIALIZED_PRE_APPROVED,
74 State.SESSION_ACCEPTED);
75 private static final long BUSY_TIME_OUT = 30;
76
77 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
78 private final Queue<
79 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
80 pendingIceCandidates = new LinkedList<>();
81 private final OmemoVerification omemoVerification = new OmemoVerification();
82 private final CallIntegration callIntegration;
83 private final Message message;
84
85 private Set<Media> proposedMedia;
86 private RtpContentMap initiatorRtpContentMap;
87 private RtpContentMap responderRtpContentMap;
88 private RtpContentMap incomingContentAdd;
89 private RtpContentMap outgoingContentAdd;
90 private IceUdpTransportInfo.Setup peerDtlsSetup;
91 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
92 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
93 private ScheduledFuture<?> ringingTimeoutFuture;
94
95 JingleRtpConnection(
96 final JingleConnectionManager jingleConnectionManager,
97 final Id id,
98 final Jid initiator) {
99 this(
100 jingleConnectionManager,
101 id,
102 initiator,
103 new CallIntegration(
104 jingleConnectionManager
105 .getXmppConnectionService()
106 .getApplicationContext()));
107 this.callIntegration.setAddress(
108 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
109 final var contact = id.getContact();
110 this.callIntegration.setCallerDisplayName(
111 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
112 this.callIntegration.setInitialized();
113 }
114
115 JingleRtpConnection(
116 final JingleConnectionManager jingleConnectionManager,
117 final Id id,
118 final Jid initiator,
119 final CallIntegration callIntegration) {
120 super(jingleConnectionManager, id, initiator);
121 final Conversation conversation =
122 jingleConnectionManager
123 .getXmppConnectionService()
124 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
125 this.message =
126 new Message(
127 conversation,
128 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
129 Message.TYPE_RTP_SESSION,
130 id.sessionId);
131 this.callIntegration = callIntegration;
132 this.callIntegration.setCallback(this);
133 }
134
135 @Override
136 synchronized void deliverPacket(final Iq iq) {
137 final var jingle = iq.getExtension(Jingle.class);
138 switch (jingle.getAction()) {
139 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
140 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
141 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
142 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
143 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
144 case CONTENT_ACCEPT -> receiveContentAccept(iq);
145 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
146 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
147 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
148 default -> {
149 respondOk(iq);
150 Log.d(
151 Config.LOGTAG,
152 String.format(
153 "%s: received unhandled jingle action %s",
154 id.account.getJid().asBareJid(), jingle.getAction()));
155 }
156 }
157 }
158
159 @Override
160 synchronized void notifyRebound() {
161 if (isTerminated()) {
162 return;
163 }
164 webRTCWrapper.close();
165 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
166 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
167 }
168 if (isInState(
169 State.SESSION_INITIALIZED,
170 State.SESSION_INITIALIZED_PRE_APPROVED,
171 State.SESSION_ACCEPTED)) {
172 // we might have already changed resources (full jid) at this point; so this might not
173 // even reach the other party
174 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
175 } else {
176 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
177 finish();
178 }
179 }
180
181 private void receiveSessionTerminate(final Iq jinglePacket) {
182 respondOk(jinglePacket);
183 final var jingle = jinglePacket.getExtension(Jingle.class);
184 final Jingle.ReasonWrapper wrapper = jingle.getReason();
185 final State previous = this.state;
186 Log.d(
187 Config.LOGTAG,
188 id.account.getJid().asBareJid()
189 + ": received session terminate reason="
190 + wrapper.reason
191 + "("
192 + Strings.nullToEmpty(wrapper.text)
193 + ") while in state "
194 + previous);
195 if (TERMINATED.contains(previous)) {
196 Log.d(
197 Config.LOGTAG,
198 id.account.getJid().asBareJid()
199 + ": ignoring session terminate because already in "
200 + previous);
201 return;
202 }
203 webRTCWrapper.close();
204 final State target = reasonToState(wrapper.reason);
205 transitionOrThrow(target);
206 writeLogMessage(target);
207 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
208 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
209 }
210 finish();
211 }
212
213 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
214 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
215 // INITIALIZED only after transport-info has been received
216 if (isInState(
217 State.NULL,
218 State.PROCEED,
219 State.SESSION_INITIALIZED,
220 State.SESSION_INITIALIZED_PRE_APPROVED,
221 State.SESSION_ACCEPTED)) {
222 final RtpContentMap contentMap;
223 try {
224 contentMap = RtpContentMap.of(jingle);
225 } catch (final IllegalArgumentException | NullPointerException e) {
226 Log.d(
227 Config.LOGTAG,
228 id.account.getJid().asBareJid()
229 + ": improperly formatted contents; ignoring",
230 e);
231 respondOk(jinglePacket);
232 return;
233 }
234 receiveTransportInfo(jinglePacket, contentMap);
235 } else {
236 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
237 }
238 }
239
240 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
241 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
242 candidates = contentMap.contents.entrySet();
243 final RtpContentMap remote = getRemoteContentMap();
244 final Set<String> remoteContentIds =
245 remote == null ? Collections.emptySet() : remote.contents.keySet();
246 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
247 Log.d(
248 Config.LOGTAG,
249 "received transport-info for unknown contents "
250 + contentMap.contents.keySet()
251 + " (known: "
252 + remoteContentIds
253 + ")");
254 respondOk(jinglePacket);
255 pendingIceCandidates.addAll(candidates);
256 return;
257 }
258 if (this.state != State.SESSION_ACCEPTED) {
259 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
260 respondOk(jinglePacket);
261 pendingIceCandidates.addAll(candidates);
262 return;
263 }
264 // zero candidates + modified credentials are an ICE restart offer
265 if (checkForIceRestart(jinglePacket, contentMap)) {
266 return;
267 }
268 respondOk(jinglePacket);
269 try {
270 processCandidates(candidates);
271 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
272 Log.w(
273 Config.LOGTAG,
274 id.account.getJid().asBareJid()
275 + ": PeerConnection was not initialized when processing transport info."
276 + " this usually indicates a race condition that can be ignored");
277 }
278 }
279
280 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
281 final RtpContentMap modification;
282 try {
283 modification = RtpContentMap.of(jingle);
284 modification.requireContentDescriptions();
285 } catch (final RuntimeException e) {
286 Log.d(
287 Config.LOGTAG,
288 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
289 Throwables.getRootCause(e));
290 respondOk(iq);
291 webRTCWrapper.close();
292 sendSessionTerminate(Reason.of(e), e.getMessage());
293 return;
294 }
295 if (isInState(State.SESSION_ACCEPTED)) {
296 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
297 final ListenableFuture<RtpContentMap> future =
298 receiveRtpContentMap(
299 modification,
300 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
301 Futures.addCallback(
302 future,
303 new FutureCallback<>() {
304 @Override
305 public void onSuccess(final RtpContentMap rtpContentMap) {
306 receiveContentAdd(iq, rtpContentMap);
307 }
308
309 @Override
310 public void onFailure(@NonNull Throwable throwable) {
311 respondOk(iq);
312 final Throwable rootCause = Throwables.getRootCause(throwable);
313 Log.d(
314 Config.LOGTAG,
315 id.account.getJid().asBareJid()
316 + ": improperly formatted contents in content-add",
317 throwable);
318 webRTCWrapper.close();
319 sendSessionTerminate(
320 Reason.ofThrowable(rootCause), rootCause.getMessage());
321 }
322 },
323 MoreExecutors.directExecutor());
324 } else {
325 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
326 }
327 }
328
329 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
330 final RtpContentMap remote = getRemoteContentMap();
331 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
332 respondOk(jinglePacket);
333 this.webRTCWrapper.close();
334 sendSessionTerminate(
335 Reason.FAILED_APPLICATION,
336 String.format(
337 "contents with names %s already exists",
338 Joiner.on(", ").join(modification.getNames())));
339 return;
340 }
341 final ContentAddition contentAddition =
342 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
343
344 final RtpContentMap outgoing = this.outgoingContentAdd;
345 final Set<ContentAddition.Summary> outgoingContentAddSummary =
346 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
347
348 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
349 if (isInitiator()) {
350 Log.d(
351 Config.LOGTAG,
352 id.getAccount().getJid().asBareJid()
353 + ": respond with tie break to matching content-add offer");
354 respondWithTieBreak(jinglePacket);
355 } else {
356 Log.d(
357 Config.LOGTAG,
358 id.getAccount().getJid().asBareJid()
359 + ": automatically accept matching content-add offer");
360 acceptContentAdd(contentAddition.summary, modification);
361 }
362 return;
363 }
364
365 // once we can display multiple video tracks we can be more loose with this condition
366 // theoretically it should also be fine to automatically accept audio only contents
367 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
368 Log.d(
369 Config.LOGTAG,
370 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
371 this.incomingContentAdd = modification;
372 respondOk(jinglePacket);
373 updateEndUserState();
374 } else {
375 respondOk(jinglePacket);
376 // TODO do we want to add a reason?
377 rejectContentAdd(modification);
378 }
379 }
380
381 private void receiveContentAccept(final Iq jinglePacket) {
382 final var jingle = jinglePacket.getExtension(Jingle.class);
383 final RtpContentMap receivedContentAccept;
384 try {
385 receivedContentAccept = RtpContentMap.of(jingle);
386 receivedContentAccept.requireContentDescriptions();
387 } catch (final RuntimeException e) {
388 Log.d(
389 Config.LOGTAG,
390 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
391 Throwables.getRootCause(e));
392 respondOk(jinglePacket);
393 webRTCWrapper.close();
394 sendSessionTerminate(Reason.of(e), e.getMessage());
395 return;
396 }
397
398 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
399 if (outgoingContentAdd == null) {
400 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
401 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
402 return;
403 }
404 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
405 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
406 this.outgoingContentAdd = null;
407 respondOk(jinglePacket);
408 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
409 final ListenableFuture<RtpContentMap> future =
410 receiveRtpContentMap(
411 receivedContentAccept,
412 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
413 Futures.addCallback(
414 future,
415 new FutureCallback<>() {
416 @Override
417 public void onSuccess(final RtpContentMap result) {
418 receiveContentAccept(result);
419 }
420
421 @Override
422 public void onFailure(@NonNull final Throwable throwable) {
423 webRTCWrapper.close();
424 sendSessionTerminate(
425 Reason.ofThrowable(throwable), throwable.getMessage());
426 }
427 },
428 MoreExecutors.directExecutor());
429 } else {
430 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
431 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
432 }
433 }
434
435 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
436 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
437 final RtpContentMap modifiedContentMap =
438 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
439
440 setRemoteContentMap(modifiedContentMap);
441
442 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
443
444 final org.webrtc.SessionDescription sdp =
445 new org.webrtc.SessionDescription(
446 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
447
448 try {
449 this.webRTCWrapper.setRemoteDescription(sdp).get();
450 } catch (final Exception e) {
451 final Throwable cause = Throwables.getRootCause(e);
452 Log.d(
453 Config.LOGTAG,
454 id.getAccount().getJid().asBareJid()
455 + ": unable to set remote description after receiving content-accept",
456 cause);
457 webRTCWrapper.close();
458 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
459 return;
460 }
461 Log.d(
462 Config.LOGTAG,
463 id.getAccount().getJid().asBareJid()
464 + ": remote has accepted content-add "
465 + ContentAddition.summary(receivedContentAccept));
466 processCandidates(receivedContentAccept.contents.entrySet());
467 updateEndUserState();
468 }
469
470 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
471 if (this.state != State.SESSION_ACCEPTED) {
472 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
473 return;
474 }
475 final Map<String, Content.Senders> modification =
476 Maps.transformEntries(
477 jingle.getJingleContents(), (key, value) -> value.getSenders());
478 final boolean isInitiator = isInitiator();
479 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
480 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
481 final Set<String> currentOutgoingMediaIds =
482 currentOutgoing == null
483 ? Collections.emptySet()
484 : currentOutgoing.contents.keySet();
485 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
486 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
487 respondOk(jinglePacket);
488 final RtpContentMap modifiedContentMap;
489 try {
490 modifiedContentMap =
491 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
492 } catch (final IllegalArgumentException e) {
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
495 return;
496 }
497 this.outgoingContentAdd = modifiedContentMap;
498 Log.d(
499 Config.LOGTAG,
500 id.account.getJid().asBareJid()
501 + ": processed content-modification for pending content-add");
502 } else if (remoteContentMap != null
503 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
504 respondOk(jinglePacket);
505 final RtpContentMap modifiedRemoteContentMap;
506 try {
507 modifiedRemoteContentMap =
508 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
509 } catch (final IllegalArgumentException e) {
510 webRTCWrapper.close();
511 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
512 return;
513 }
514 final SessionDescription offer;
515 try {
516 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
517 } catch (final IllegalArgumentException | NullPointerException e) {
518 Log.d(
519 Config.LOGTAG,
520 id.getAccount().getJid().asBareJid()
521 + ": unable convert offer from content-modify to SDP",
522 e);
523 webRTCWrapper.close();
524 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
525 return;
526 }
527 Log.d(
528 Config.LOGTAG,
529 id.account.getJid().asBareJid() + ": auto accepting content-modification");
530 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
531 } else {
532 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
533 respondWithItemNotFound(jinglePacket);
534 }
535 }
536
537 private void autoAcceptContentModify(
538 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
539 this.setRemoteContentMap(modifiedRemoteContentMap);
540 final org.webrtc.SessionDescription sdp =
541 new org.webrtc.SessionDescription(
542 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
543 try {
544 this.webRTCWrapper.setRemoteDescription(sdp).get();
545 // auto accept is only done when we already have tracks
546 final SessionDescription answer = setLocalSessionDescription();
547 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
548 modifyLocalContentMap(rtpContentMap);
549 // we do not need to send an answer but do we have to resend the candidates currently in
550 // SDP?
551 // resendCandidatesFromSdp(answer);
552 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
553 } catch (final Exception e) {
554 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.FAILED_APPLICATION);
557 }
558 }
559
560 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
561 final SessionDescription answer) {
562 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
563 new ImmutableMultimap.Builder<>();
564 for (final SessionDescription.Media media : answer.media) {
565 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
566 if (Strings.isNullOrEmpty(mid)) {
567 continue;
568 }
569 for (final String sdpCandidate : media.attributes.get("candidate")) {
570 final IceUdpTransportInfo.Candidate candidate =
571 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
572 if (candidate != null) {
573 candidateBuilder.put(mid, candidate);
574 }
575 }
576 }
577 return candidateBuilder.build();
578 }
579
580 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
581 final RtpContentMap receivedContentReject;
582 try {
583 receivedContentReject = RtpContentMap.of(jingle);
584 } catch (final RuntimeException e) {
585 Log.d(
586 Config.LOGTAG,
587 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
588 Throwables.getRootCause(e));
589 respondOk(jinglePacket);
590 this.webRTCWrapper.close();
591 sendSessionTerminate(Reason.of(e), e.getMessage());
592 return;
593 }
594
595 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
596 if (outgoingContentAdd == null) {
597 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
598 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
599 return;
600 }
601 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
602 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
603 this.outgoingContentAdd = null;
604 respondOk(jinglePacket);
605 Log.d(Config.LOGTAG, jinglePacket.toString());
606 receiveContentReject(ourSummary);
607 } else {
608 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
609 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
610 }
611 }
612
613 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
614 try {
615 this.webRTCWrapper.removeTrack(Media.VIDEO);
616 final RtpContentMap localContentMap = customRollback();
617 modifyLocalContentMap(localContentMap);
618 } catch (final Exception e) {
619 final Throwable cause = Throwables.getRootCause(e);
620 Log.d(
621 Config.LOGTAG,
622 id.getAccount().getJid().asBareJid()
623 + ": unable to rollback local description after receiving"
624 + " content-reject",
625 cause);
626 webRTCWrapper.close();
627 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
628 return;
629 }
630 Log.d(
631 Config.LOGTAG,
632 id.getAccount().getJid().asBareJid()
633 + ": remote has rejected our content-add "
634 + summary);
635 }
636
637 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
638 final RtpContentMap receivedContentRemove;
639 try {
640 receivedContentRemove = RtpContentMap.of(jingle);
641 receivedContentRemove.requireContentDescriptions();
642 } catch (final RuntimeException e) {
643 Log.d(
644 Config.LOGTAG,
645 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
646 Throwables.getRootCause(e));
647 respondOk(jinglePacket);
648 this.webRTCWrapper.close();
649 sendSessionTerminate(Reason.of(e), e.getMessage());
650 return;
651 }
652 respondOk(jinglePacket);
653 receiveContentRemove(receivedContentRemove);
654 }
655
656 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
657 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
658 final Set<ContentAddition.Summary> contentAddSummary =
659 incomingContentAdd == null
660 ? Collections.emptySet()
661 : ContentAddition.summary(incomingContentAdd);
662 final Set<ContentAddition.Summary> removeSummary =
663 ContentAddition.summary(receivedContentRemove);
664 if (contentAddSummary.equals(removeSummary)) {
665 this.incomingContentAdd = null;
666 updateEndUserState();
667 } else {
668 webRTCWrapper.close();
669 sendSessionTerminate(
670 Reason.FAILED_APPLICATION,
671 String.format(
672 "%s only supports %s as a means to retract a not yet accepted %s",
673 BuildConfig.APP_NAME,
674 Jingle.Action.CONTENT_REMOVE,
675 Jingle.Action.CONTENT_ADD));
676 }
677 }
678
679 public synchronized void retractContentAdd() {
680 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
681 if (outgoingContentAdd == null) {
682 throw new IllegalStateException("Not outgoing content add");
683 }
684 try {
685 webRTCWrapper.removeTrack(Media.VIDEO);
686 final RtpContentMap localContentMap = customRollback();
687 modifyLocalContentMap(localContentMap);
688 } catch (final Exception e) {
689 final Throwable cause = Throwables.getRootCause(e);
690 Log.d(
691 Config.LOGTAG,
692 id.getAccount().getJid().asBareJid()
693 + ": unable to rollback local description after trying to retract"
694 + " content-add",
695 cause);
696 webRTCWrapper.close();
697 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
698 return;
699 }
700 this.outgoingContentAdd = null;
701 final Iq retract =
702 outgoingContentAdd
703 .toStub()
704 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
705 this.send(retract);
706 Log.d(
707 Config.LOGTAG,
708 id.getAccount().getJid()
709 + ": retract content-add "
710 + ContentAddition.summary(outgoingContentAdd));
711 }
712
713 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
714 final SessionDescription sdp = setLocalSessionDescription();
715 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
716 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
717 this.webRTCWrapper
718 .setRemoteDescription(
719 new org.webrtc.SessionDescription(
720 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
721 .get();
722 return localRtpContentMap;
723 }
724
725 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
726 final RtpContentMap currentRemote = getRemoteContentMap();
727 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
728 if (diff.isEmpty()) {
729 throw new IllegalStateException(
730 "Unexpected rollback condition. No difference between local and remote");
731 }
732 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
733 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
734 final RtpContentMap nextRemote =
735 currentRemote.addContent(
736 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
737 return SessionDescription.of(nextRemote, isResponder());
738 }
739 throw new IllegalStateException(
740 "Unexpected rollback condition. Senders were not uniformly none");
741 }
742
743 public synchronized void acceptContentAdd(
744 @NonNull final Set<ContentAddition.Summary> contentAddition) {
745 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
746 if (incomingContentAdd == null) {
747 throw new IllegalStateException("No incoming content add");
748 }
749
750 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
751 this.incomingContentAdd = null;
752 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
753 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
754 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
755 Log.d(
756 Config.LOGTAG,
757 "content addition is receive only. we want to upgrade to 'both'");
758 final RtpContentMap modifiedSenders =
759 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
760 final Iq proposedContentModification =
761 modifiedSenders
762 .toStub()
763 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
764 proposedContentModification.setTo(id.with);
765 xmppConnectionService.sendIqPacket(
766 id.account,
767 proposedContentModification,
768 (response) -> {
769 if (response.getType() == Iq.Type.RESULT) {
770 Log.d(
771 Config.LOGTAG,
772 id.account.getJid().asBareJid()
773 + ": remote has accepted our upgrade to"
774 + " senders=both");
775 acceptContentAdd(
776 ContentAddition.summary(modifiedSenders), modifiedSenders);
777 } else {
778 Log.d(
779 Config.LOGTAG,
780 id.account.getJid().asBareJid()
781 + ": remote has rejected our upgrade to"
782 + " senders=both");
783 acceptContentAdd(contentAddition, incomingContentAdd);
784 }
785 });
786 } else {
787 acceptContentAdd(contentAddition, incomingContentAdd);
788 }
789 } else {
790 throw new IllegalStateException(
791 "Accepted content add does not match pending content-add");
792 }
793 }
794
795 private void acceptContentAdd(
796 @NonNull final Set<ContentAddition.Summary> contentAddition,
797 final RtpContentMap incomingContentAdd) {
798 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
799 final RtpContentMap modifiedContentMap =
800 getRemoteContentMap().addContent(incomingContentAdd, setup);
801 this.setRemoteContentMap(modifiedContentMap);
802
803 final SessionDescription offer;
804 try {
805 offer = SessionDescription.of(modifiedContentMap, isResponder());
806 } catch (final IllegalArgumentException | NullPointerException e) {
807 Log.d(
808 Config.LOGTAG,
809 id.getAccount().getJid().asBareJid()
810 + ": unable convert offer from content-add to SDP",
811 e);
812 webRTCWrapper.close();
813 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
814 return;
815 }
816 this.incomingContentAdd = null;
817 acceptContentAdd(contentAddition, offer);
818 }
819
820 private void acceptContentAdd(
821 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
822 final org.webrtc.SessionDescription sdp =
823 new org.webrtc.SessionDescription(
824 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
825 try {
826 this.webRTCWrapper.setRemoteDescription(sdp).get();
827
828 // TODO add tracks for 'media' where contentAddition.senders matches
829
830 // TODO if senders.sending(isInitiator())
831
832 this.webRTCWrapper.addTrack(Media.VIDEO);
833
834 // TODO add additional transceivers for recv only cases
835
836 final SessionDescription answer = setLocalSessionDescription();
837 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
838
839 final RtpContentMap contentAcceptMap =
840 rtpContentMap.toContentModification(
841 Collections2.transform(contentAddition, ca -> ca.name));
842
843 Log.d(
844 Config.LOGTAG,
845 id.getAccount().getJid().asBareJid()
846 + ": sending content-accept "
847 + ContentAddition.summary(contentAcceptMap));
848
849 addIceCandidatesFromBlackLog();
850
851 modifyLocalContentMap(rtpContentMap);
852 final ListenableFuture<RtpContentMap> future =
853 prepareOutgoingContentMap(contentAcceptMap);
854 Futures.addCallback(
855 future,
856 new FutureCallback<>() {
857 @Override
858 public void onSuccess(final RtpContentMap rtpContentMap) {
859 sendContentAccept(rtpContentMap);
860 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
861 }
862
863 @Override
864 public void onFailure(@NonNull final Throwable throwable) {
865 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
866 }
867 },
868 MoreExecutors.directExecutor());
869 } catch (final Exception e) {
870 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
871 webRTCWrapper.close();
872 sendSessionTerminate(Reason.FAILED_APPLICATION);
873 }
874 }
875
876 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
877 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
878 send(iq);
879 }
880
881 public synchronized void rejectContentAdd() {
882 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
883 if (incomingContentAdd == null) {
884 throw new IllegalStateException("No incoming content add");
885 }
886 this.incomingContentAdd = null;
887 updateEndUserState();
888 rejectContentAdd(incomingContentAdd);
889 }
890
891 private void rejectContentAdd(final RtpContentMap contentMap) {
892 final Iq iq =
893 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
894 Log.d(
895 Config.LOGTAG,
896 id.getAccount().getJid().asBareJid()
897 + ": rejecting content "
898 + ContentAddition.summary(contentMap));
899 send(iq);
900 }
901
902 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
903 final RtpContentMap existing = getRemoteContentMap();
904 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
905 final IceUdpTransportInfo.Credentials newCredentials;
906 try {
907 existingCredentials = existing.getCredentials();
908 newCredentials = rtpContentMap.getDistinctCredentials();
909 } catch (final IllegalStateException e) {
910 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
911 return false;
912 }
913 if (existingCredentials.contains(newCredentials)) {
914 return false;
915 }
916 // TODO an alternative approach is to check if we already got an iq result to our
917 // ICE-restart
918 // and if that's the case we are seeing an answer.
919 // This might be more spec compliant but also more error prone potentially
920 final boolean isSignalStateStable =
921 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
922 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
923 final boolean isOffer = rtpContentMap.emptyCandidates();
924 final RtpContentMap restartContentMap;
925 try {
926 if (isOffer) {
927 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
928 restartContentMap =
929 existing.modifiedCredentials(
930 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
931 } else {
932 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
933 Log.d(
934 Config.LOGTAG,
935 "received confirmation of ICE restart"
936 + newCredentials
937 + " peer_setup="
938 + setup);
939 // DTLS setup attribute needs to be rewritten to reflect current peer state
940 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
941 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
942 }
943 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
944 return isOffer;
945 } else {
946 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
947 respondWithTieBreak(jinglePacket);
948 return true;
949 }
950 } catch (final Exception exception) {
951 respondOk(jinglePacket);
952 final Throwable rootCause = Throwables.getRootCause(exception);
953 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
954 // If this happens a termination is already in progress
955 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
956 return true;
957 }
958 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
959 webRTCWrapper.close();
960 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
961 return true;
962 }
963 }
964
965 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
966 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
967 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
968 throw new IllegalStateException("Invalid peer setup");
969 }
970 return peerSetup;
971 }
972
973 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
974 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
975 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
976 }
977 this.peerDtlsSetup = setup;
978 }
979
980 private boolean applyIceRestart(
981 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
982 throws ExecutionException, InterruptedException {
983 final SessionDescription sessionDescription =
984 SessionDescription.of(restartContentMap, isResponder());
985 final org.webrtc.SessionDescription.Type type =
986 isOffer
987 ? org.webrtc.SessionDescription.Type.OFFER
988 : org.webrtc.SessionDescription.Type.ANSWER;
989 org.webrtc.SessionDescription sdp =
990 new org.webrtc.SessionDescription(type, sessionDescription.toString());
991 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
992 if (isInitiator()) {
993 // We ignore the offer and respond with tie-break. This will clause the responder
994 // not to apply the content map
995 return false;
996 }
997 }
998 webRTCWrapper.setRemoteDescription(sdp).get();
999 setRemoteContentMap(restartContentMap);
1000 if (isOffer) {
1001 final SessionDescription localSessionDescription = setLocalSessionDescription();
1002 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1003 // We need to respond OK before sending any candidates
1004 respondOk(jinglePacket);
1005 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1006 } else {
1007 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1008 }
1009 return true;
1010 }
1011
1012 private void processCandidates(
1013 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1014 contents) {
1015 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1016 content : contents) {
1017 processCandidate(content);
1018 }
1019 }
1020
1021 private void processCandidate(
1022 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1023 content) {
1024 final RtpContentMap rtpContentMap = getRemoteContentMap();
1025 final List<String> indices = toIdentificationTags(rtpContentMap);
1026 final String sdpMid = content.getKey(); // aka content name
1027 final IceUdpTransportInfo transport = content.getValue().transport;
1028 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1029
1030 // TODO check that credentials remained the same
1031
1032 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1033 final String sdp;
1034 try {
1035 sdp = candidate.toSdpAttribute(credentials.ufrag);
1036 } catch (final IllegalArgumentException e) {
1037 Log.d(
1038 Config.LOGTAG,
1039 id.account.getJid().asBareJid()
1040 + ": ignoring invalid ICE candidate "
1041 + e.getMessage());
1042 continue;
1043 }
1044 final int mLineIndex = indices.indexOf(sdpMid);
1045 if (mLineIndex < 0) {
1046 Log.w(
1047 Config.LOGTAG,
1048 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1049 }
1050 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1051 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1052 this.webRTCWrapper.addIceCandidate(iceCandidate);
1053 }
1054 }
1055
1056 private RtpContentMap getRemoteContentMap() {
1057 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1058 }
1059
1060 private RtpContentMap getLocalContentMap() {
1061 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1062 }
1063
1064 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1065 final Group originalGroup = rtpContentMap.group;
1066 final List<String> identificationTags =
1067 originalGroup == null
1068 ? rtpContentMap.getNames()
1069 : originalGroup.getIdentificationTags();
1070 if (identificationTags.size() == 0) {
1071 Log.w(
1072 Config.LOGTAG,
1073 id.account.getJid().asBareJid()
1074 + ": no identification tags found in initial offer. we won't be able to"
1075 + " calculate mLineIndices");
1076 }
1077 return identificationTags;
1078 }
1079
1080 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1081 final Jingle jinglePacket, final boolean expectVerification) {
1082 try {
1083 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1084 } catch (final Exception e) {
1085 return Futures.immediateFailedFuture(e);
1086 }
1087 }
1088
1089 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1090 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1091 Log.d(
1092 Config.LOGTAG,
1093 "receiveRtpContentMap("
1094 + receivedContentMap.getClass().getSimpleName()
1095 + ",expectVerification="
1096 + expectVerification
1097 + ")");
1098 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1099 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1100 id.account
1101 .getAxolotlService()
1102 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1103 return Futures.transform(
1104 future,
1105 omemoVerifiedPayload -> {
1106 // TODO test if an exception here triggers a correct abort
1107 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1108 Log.d(
1109 Config.LOGTAG,
1110 id.account.getJid().asBareJid()
1111 + ": received verifiable DTLS fingerprint via "
1112 + omemoVerification);
1113 return omemoVerifiedPayload.getPayload();
1114 },
1115 MoreExecutors.directExecutor());
1116 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1117 return Futures.immediateFailedFuture(
1118 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1119 } else {
1120 return Futures.immediateFuture(receivedContentMap);
1121 }
1122 }
1123
1124 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1125 if (isInitiator()) {
1126 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1127 return;
1128 }
1129 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1130 Futures.addCallback(
1131 future,
1132 new FutureCallback<>() {
1133 @Override
1134 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1135 receiveSessionInitiate(jinglePacket, rtpContentMap);
1136 }
1137
1138 @Override
1139 public void onFailure(@NonNull final Throwable throwable) {
1140 respondOk(jinglePacket);
1141 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1142 }
1143 },
1144 MoreExecutors.directExecutor());
1145 }
1146
1147 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1148 try {
1149 contentMap.requireContentDescriptions();
1150 contentMap.requireDTLSFingerprint(true);
1151 } catch (final RuntimeException e) {
1152 Log.d(
1153 Config.LOGTAG,
1154 id.account.getJid().asBareJid() + ": improperly formatted contents",
1155 Throwables.getRootCause(e));
1156 respondOk(jinglePacket);
1157 sendSessionTerminate(Reason.of(e), e.getMessage());
1158 return;
1159 }
1160 Log.d(
1161 Config.LOGTAG,
1162 "processing session-init with " + contentMap.contents.size() + " contents");
1163 final State target;
1164 if (this.state == State.PROCEED) {
1165 Preconditions.checkState(
1166 proposedMedia != null && proposedMedia.size() > 0,
1167 "proposed media must be set when processing pre-approved session-initiate");
1168 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1169 sendSessionTerminate(
1170 Reason.SECURITY_ERROR,
1171 String.format(
1172 "Your session proposal (Jingle Message Initiation) included media"
1173 + " %s but your session-initiate was %s",
1174 this.proposedMedia, contentMap.getMedia()));
1175 return;
1176 }
1177 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1178 } else {
1179 target = State.SESSION_INITIALIZED;
1180 setProposedMedia(contentMap.getMedia());
1181 }
1182 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1183 respondOk(jinglePacket);
1184 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1185 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1186 Log.d(
1187 Config.LOGTAG,
1188 id.account.getJid().asBareJid()
1189 + ": automatically accepting session-initiate");
1190 sendSessionAccept();
1191 } else {
1192 Log.d(
1193 Config.LOGTAG,
1194 id.account.getJid().asBareJid()
1195 + ": received not pre-approved session-initiate. start ringing");
1196 startRinging();
1197 }
1198 } else {
1199 Log.d(
1200 Config.LOGTAG,
1201 String.format(
1202 "%s: received session-initiate while in state %s",
1203 id.account.getJid().asBareJid(), state));
1204 terminateWithOutOfOrder(jinglePacket);
1205 }
1206 }
1207
1208 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1209 if (isResponder()) {
1210 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1211 return;
1212 }
1213 final ListenableFuture<RtpContentMap> future =
1214 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1215 Futures.addCallback(
1216 future,
1217 new FutureCallback<>() {
1218 @Override
1219 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1220 receiveSessionAccept(jinglePacket, rtpContentMap);
1221 }
1222
1223 @Override
1224 public void onFailure(@NonNull final Throwable throwable) {
1225 respondOk(jinglePacket);
1226 Log.d(
1227 Config.LOGTAG,
1228 id.account.getJid().asBareJid()
1229 + ": improperly formatted contents in session-accept",
1230 throwable);
1231 webRTCWrapper.close();
1232 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1233 }
1234 },
1235 MoreExecutors.directExecutor());
1236 }
1237
1238 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1239 try {
1240 contentMap.requireContentDescriptions();
1241 contentMap.requireDTLSFingerprint();
1242 } catch (final RuntimeException e) {
1243 respondOk(jinglePacket);
1244 Log.d(
1245 Config.LOGTAG,
1246 id.account.getJid().asBareJid()
1247 + ": improperly formatted contents in session-accept",
1248 e);
1249 webRTCWrapper.close();
1250 sendSessionTerminate(Reason.of(e), e.getMessage());
1251 return;
1252 }
1253 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1254 if (!initiatorMedia.equals(contentMap.getMedia())) {
1255 sendSessionTerminate(
1256 Reason.SECURITY_ERROR,
1257 String.format(
1258 "Your session-included included media %s but our session-initiate was"
1259 + " %s",
1260 this.proposedMedia, contentMap.getMedia()));
1261 return;
1262 }
1263 Log.d(
1264 Config.LOGTAG,
1265 "processing session-accept with " + contentMap.contents.size() + " contents");
1266 if (transition(State.SESSION_ACCEPTED)) {
1267 respondOk(jinglePacket);
1268 receiveSessionAccept(contentMap);
1269 } else {
1270 Log.d(
1271 Config.LOGTAG,
1272 String.format(
1273 "%s: received session-accept while in state %s",
1274 id.account.getJid().asBareJid(), state));
1275 respondOk(jinglePacket);
1276 }
1277 }
1278
1279 private void receiveSessionAccept(final RtpContentMap contentMap) {
1280 this.responderRtpContentMap = contentMap;
1281 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1282 final SessionDescription sessionDescription;
1283 try {
1284 sessionDescription = SessionDescription.of(contentMap, false);
1285 } catch (final IllegalArgumentException | NullPointerException e) {
1286 Log.d(
1287 Config.LOGTAG,
1288 id.account.getJid().asBareJid()
1289 + ": unable convert offer from session-accept to SDP",
1290 e);
1291 webRTCWrapper.close();
1292 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1293 return;
1294 }
1295 final org.webrtc.SessionDescription answer =
1296 new org.webrtc.SessionDescription(
1297 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1298 try {
1299 this.webRTCWrapper.setRemoteDescription(answer).get();
1300 } catch (final Exception e) {
1301 Log.d(
1302 Config.LOGTAG,
1303 id.account.getJid().asBareJid()
1304 + ": unable to set remote description after receiving session-accept",
1305 Throwables.getRootCause(e));
1306 webRTCWrapper.close();
1307 sendSessionTerminate(
1308 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1309 return;
1310 }
1311 processCandidates(contentMap.contents.entrySet());
1312 }
1313
1314 private void sendSessionAccept() {
1315 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1316 if (rtpContentMap == null) {
1317 throw new IllegalStateException("initiator RTP Content Map has not been set");
1318 }
1319 final SessionDescription offer;
1320 try {
1321 offer = SessionDescription.of(rtpContentMap, true);
1322 } catch (final IllegalArgumentException | NullPointerException e) {
1323 Log.d(
1324 Config.LOGTAG,
1325 id.account.getJid().asBareJid()
1326 + ": unable convert offer from session-initiate to SDP",
1327 e);
1328 webRTCWrapper.close();
1329 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1330 return;
1331 }
1332 sendSessionAccept(rtpContentMap.getMedia(), offer);
1333 }
1334
1335 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1336 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1337 }
1338
1339 private synchronized void sendSessionAccept(
1340 final Set<Media> media,
1341 final SessionDescription offer,
1342 final List<PeerConnection.IceServer> iceServers) {
1343 if (isTerminated()) {
1344 Log.w(
1345 Config.LOGTAG,
1346 id.account.getJid().asBareJid()
1347 + ": ICE servers got discovered when session was already terminated."
1348 + " nothing to do.");
1349 return;
1350 }
1351 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1352 try {
1353 setupWebRTC(media, iceServers, !includeCandidates);
1354 } catch (final WebRTCWrapper.InitializationException e) {
1355 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1356 webRTCWrapper.close();
1357 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1358 return;
1359 }
1360 final org.webrtc.SessionDescription sdp =
1361 new org.webrtc.SessionDescription(
1362 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1363 try {
1364 this.webRTCWrapper.setRemoteDescription(sdp).get();
1365 addIceCandidatesFromBlackLog();
1366 org.webrtc.SessionDescription webRTCSessionDescription =
1367 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1368 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1369 } catch (final Exception e) {
1370 failureToAcceptSession(e);
1371 }
1372 }
1373
1374 private void failureToAcceptSession(final Throwable throwable) {
1375 if (isTerminated()) {
1376 return;
1377 }
1378 final Throwable rootCause = Throwables.getRootCause(throwable);
1379 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1380 webRTCWrapper.close();
1381 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1382 }
1383
1384 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1385 if (isTerminated()) {
1386 return;
1387 }
1388 final Throwable rootCause = Throwables.getRootCause(throwable);
1389 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1390 webRTCWrapper.close();
1391 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1392 }
1393
1394 private void addIceCandidatesFromBlackLog() {
1395 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1396 while ((foo = this.pendingIceCandidates.poll()) != null) {
1397 processCandidate(foo);
1398 Log.d(
1399 Config.LOGTAG,
1400 id.account.getJid().asBareJid() + ": added candidate from back log");
1401 }
1402 }
1403
1404 private void prepareSessionAccept(
1405 final org.webrtc.SessionDescription webRTCSessionDescription,
1406 final boolean includeCandidates) {
1407 final SessionDescription sessionDescription =
1408 SessionDescription.parse(webRTCSessionDescription.description);
1409 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1410 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1411 if (includeCandidates) {
1412 candidates = parseCandidates(sessionDescription);
1413 } else {
1414 candidates = ImmutableMultimap.of();
1415 }
1416 this.responderRtpContentMap = respondingRtpContentMap;
1417 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1418 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1419 prepareOutgoingContentMap(respondingRtpContentMap);
1420 Futures.addCallback(
1421 outgoingContentMapFuture,
1422 new FutureCallback<>() {
1423 @Override
1424 public void onSuccess(final RtpContentMap outgoingContentMap) {
1425 if (includeCandidates) {
1426 Log.d(
1427 Config.LOGTAG,
1428 "including "
1429 + candidates.size()
1430 + " candidates in session accept");
1431 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1432 } else {
1433 sendSessionAccept(outgoingContentMap);
1434 }
1435 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1436 }
1437
1438 @Override
1439 public void onFailure(@NonNull Throwable throwable) {
1440 failureToAcceptSession(throwable);
1441 }
1442 },
1443 MoreExecutors.directExecutor());
1444 }
1445
1446 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1447 if (isTerminated()) {
1448 Log.w(
1449 Config.LOGTAG,
1450 id.account.getJid().asBareJid()
1451 + ": preparing session accept was too slow. already terminated. nothing"
1452 + " to do.");
1453 return;
1454 }
1455 transitionOrThrow(State.SESSION_ACCEPTED);
1456 final Iq sessionAccept =
1457 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1458 send(sessionAccept);
1459 }
1460
1461 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1462 final RtpContentMap rtpContentMap) {
1463 if (this.omemoVerification.hasDeviceId()) {
1464 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1465 verifiedPayloadFuture =
1466 id.account
1467 .getAxolotlService()
1468 .encrypt(
1469 rtpContentMap,
1470 id.with,
1471 omemoVerification.getDeviceId());
1472 return Futures.transform(
1473 verifiedPayloadFuture,
1474 verifiedPayload -> {
1475 omemoVerification.setOrEnsureEqual(verifiedPayload);
1476 return verifiedPayload.getPayload();
1477 },
1478 MoreExecutors.directExecutor());
1479 } else {
1480 return Futures.immediateFuture(rtpContentMap);
1481 }
1482 }
1483
1484 synchronized void deliveryMessage(
1485 final Jid from,
1486 final Element message,
1487 final String serverMessageId,
1488 final long timestamp) {
1489 Log.d(
1490 Config.LOGTAG,
1491 id.account.getJid().asBareJid()
1492 + ": delivered message to JingleRtpConnection "
1493 + message);
1494 switch (message.getName()) {
1495 case "propose" ->
1496 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1497 case "proceed" ->
1498 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1499 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1500 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1501 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1502 }
1503 }
1504
1505 void deliverFailedProceed(final String message) {
1506 Log.d(
1507 Config.LOGTAG,
1508 id.account.getJid().asBareJid()
1509 + ": receive message error for proceed message ("
1510 + Strings.nullToEmpty(message)
1511 + ")");
1512 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1513 webRTCWrapper.close();
1514 Log.d(
1515 Config.LOGTAG,
1516 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1517 this.finish();
1518 }
1519 }
1520
1521 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1522 final boolean originatedFromMyself =
1523 from.asBareJid().equals(id.account.getJid().asBareJid());
1524 if (originatedFromMyself) {
1525 if (transition(State.ACCEPTED)) {
1526 acceptedOnOtherDevice(serverMsgId, timestamp);
1527 } else {
1528 Log.d(
1529 Config.LOGTAG,
1530 id.account.getJid().asBareJid()
1531 + ": unable to transition to accept because already in state="
1532 + this.state);
1533 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1534 }
1535 } else {
1536 Log.d(
1537 Config.LOGTAG,
1538 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1539 }
1540 }
1541
1542 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1543 if (serverMsgId != null) {
1544 this.message.setServerMsgId(serverMsgId);
1545 }
1546 this.message.setTime(timestamp);
1547 this.message.setCarbon(true); // indicate that call was accepted on other device
1548 this.writeLogMessageSuccess(0);
1549 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1550 this.finish();
1551 }
1552
1553 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1554 final boolean originatedFromMyself =
1555 from.asBareJid().equals(id.account.getJid().asBareJid());
1556 // reject from another one of my clients
1557 if (originatedFromMyself) {
1558 receiveRejectFromMyself(serverMsgId, timestamp);
1559 } else if (isInitiator()) {
1560 if (from.equals(id.with)) {
1561 receiveRejectFromResponder();
1562 } else {
1563 Log.d(
1564 Config.LOGTAG,
1565 id.account.getJid()
1566 + ": ignoring reject from "
1567 + from
1568 + " for session with "
1569 + id.with);
1570 }
1571 } else {
1572 Log.d(
1573 Config.LOGTAG,
1574 id.account.getJid()
1575 + ": ignoring reject from "
1576 + from
1577 + " for session with "
1578 + id.with);
1579 }
1580 }
1581
1582 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1583 if (transition(State.REJECTED)) {
1584 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1585 this.finish();
1586 if (serverMsgId != null) {
1587 this.message.setServerMsgId(serverMsgId);
1588 }
1589 this.message.setTime(timestamp);
1590 this.message.setCarbon(true); // indicate that call was rejected on other device
1591 writeLogMessageMissed();
1592 } else {
1593 Log.d(
1594 Config.LOGTAG,
1595 "not able to transition into REJECTED because already in " + this.state);
1596 }
1597 }
1598
1599 private void receiveRejectFromResponder() {
1600 if (isInState(State.PROCEED)) {
1601 Log.d(
1602 Config.LOGTAG,
1603 id.account.getJid()
1604 + ": received reject while still in proceed. callee reconsidered");
1605 closeTransitionLogFinish(State.REJECTED_RACED);
1606 return;
1607 }
1608 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1609 Log.d(
1610 Config.LOGTAG,
1611 id.account.getJid()
1612 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee"
1613 + " reconsidered before receiving session-init");
1614 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1615 return;
1616 }
1617 Log.d(
1618 Config.LOGTAG,
1619 id.account.getJid()
1620 + ": ignoring reject from responder because already in state "
1621 + this.state);
1622 }
1623
1624 private void receivePropose(
1625 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1626 final boolean originatedFromMyself =
1627 from.asBareJid().equals(id.account.getJid().asBareJid());
1628 if (originatedFromMyself) {
1629 Log.d(
1630 Config.LOGTAG,
1631 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1632 } else if (transition(
1633 State.PROPOSED,
1634 () -> {
1635 final Collection<RtpDescription> descriptions =
1636 Collections2.transform(
1637 Collections2.filter(
1638 propose.getDescriptions(),
1639 d -> d instanceof RtpDescription),
1640 input -> (RtpDescription) input);
1641 final Collection<Media> media =
1642 Collections2.transform(descriptions, RtpDescription::getMedia);
1643 Preconditions.checkState(
1644 !media.contains(Media.UNKNOWN),
1645 "RTP descriptions contain unknown media");
1646 Log.d(
1647 Config.LOGTAG,
1648 id.account.getJid().asBareJid()
1649 + ": received session proposal from "
1650 + from
1651 + " for "
1652 + media);
1653 this.setProposedMedia(Sets.newHashSet(media));
1654 })) {
1655 if (serverMsgId != null) {
1656 this.message.setServerMsgId(serverMsgId);
1657 }
1658 this.message.setTime(timestamp);
1659 startRinging();
1660 // in environments where we always use discovery timeouts we always want to respond with
1661 // 'ringing'
1662 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1663 || id.getContact().showInContactList()) {
1664 sendJingleMessage("ringing");
1665 }
1666 } else {
1667 Log.d(
1668 Config.LOGTAG,
1669 id.account.getJid()
1670 + ": ignoring session proposal because already in "
1671 + state);
1672 }
1673 }
1674
1675 private void startRinging() {
1676 this.callIntegration.setRinging();
1677 Log.d(
1678 Config.LOGTAG,
1679 id.account.getJid().asBareJid()
1680 + ": received call from "
1681 + id.with
1682 + ". start ringing");
1683 ringingTimeoutFuture =
1684 jingleConnectionManager.schedule(
1685 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1686 if (CallIntegration.selfManaged(xmppConnectionService)) {
1687 return;
1688 }
1689 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1690 }
1691
1692 private synchronized void ringingTimeout() {
1693 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1694 switch (this.state) {
1695 case PROPOSED -> {
1696 message.markUnread();
1697 rejectCallFromProposed();
1698 }
1699 case SESSION_INITIALIZED -> {
1700 message.markUnread();
1701 rejectCallFromSessionInitiate();
1702 }
1703 }
1704 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1705 }
1706
1707 private void cancelRingingTimeout() {
1708 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1709 if (future != null && !future.isCancelled()) {
1710 future.cancel(false);
1711 }
1712 }
1713
1714 private void receiveProceed(
1715 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1716 final Set<Media> media =
1717 Preconditions.checkNotNull(
1718 this.proposedMedia, "Proposed media has to be set before handling proceed");
1719 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1720 if (from.equals(id.with)) {
1721 if (isInitiator()) {
1722 if (transition(State.PROCEED)) {
1723 if (serverMsgId != null) {
1724 this.message.setServerMsgId(serverMsgId);
1725 }
1726 this.message.setTime(timestamp);
1727 final Integer remoteDeviceId = proceed.getDeviceId();
1728 if (isOmemoEnabled()) {
1729 this.omemoVerification.setDeviceId(remoteDeviceId);
1730 } else {
1731 if (remoteDeviceId != null) {
1732 Log.d(
1733 Config.LOGTAG,
1734 id.account.getJid().asBareJid()
1735 + ": remote party signaled support for OMEMO"
1736 + " verification but we have OMEMO disabled");
1737 }
1738 this.omemoVerification.setDeviceId(null);
1739 }
1740 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1741 } else {
1742 Log.d(
1743 Config.LOGTAG,
1744 String.format(
1745 "%s: ignoring proceed because already in %s",
1746 id.account.getJid().asBareJid(), this.state));
1747 }
1748 } else {
1749 Log.d(
1750 Config.LOGTAG,
1751 String.format(
1752 "%s: ignoring proceed because we were not initializing",
1753 id.account.getJid().asBareJid()));
1754 }
1755 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1756 if (transition(State.ACCEPTED)) {
1757 Log.d(
1758 Config.LOGTAG,
1759 id.account.getJid().asBareJid()
1760 + ": moved session with "
1761 + id.with
1762 + " into state accepted after received carbon copied proceed");
1763 acceptedOnOtherDevice(serverMsgId, timestamp);
1764 }
1765 } else {
1766 Log.d(
1767 Config.LOGTAG,
1768 String.format(
1769 "%s: ignoring proceed from %s. was expected from %s",
1770 id.account.getJid().asBareJid(), from, id.with));
1771 }
1772 }
1773
1774 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1775 if (from.equals(id.with)) {
1776 final State target =
1777 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1778 if (transition(target)) {
1779 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1780 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1781 Log.d(
1782 Config.LOGTAG,
1783 id.account.getJid().asBareJid()
1784 + ": session with "
1785 + id.with
1786 + " has been retracted (serverMsgId="
1787 + serverMsgId
1788 + ")");
1789 if (serverMsgId != null) {
1790 this.message.setServerMsgId(serverMsgId);
1791 }
1792 this.message.setTime(timestamp);
1793 if (target == State.RETRACTED) {
1794 this.message.markUnread();
1795 }
1796 writeLogMessageMissed();
1797 finish();
1798 } else {
1799 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1800 }
1801 } else {
1802 // TODO parse retract from self
1803 Log.d(
1804 Config.LOGTAG,
1805 id.account.getJid().asBareJid()
1806 + ": received retract from "
1807 + from
1808 + ". expected retract from"
1809 + id.with
1810 + ". ignoring");
1811 }
1812 }
1813
1814 public void sendSessionInitiate() {
1815 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1816 }
1817
1818 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1819 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1820 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1821 }
1822
1823 private synchronized void sendSessionInitiate(
1824 final Set<Media> media,
1825 final State targetState,
1826 final List<PeerConnection.IceServer> iceServers) {
1827 if (isTerminated()) {
1828 Log.w(
1829 Config.LOGTAG,
1830 id.account.getJid().asBareJid()
1831 + ": ICE servers got discovered when session was already terminated."
1832 + " nothing to do.");
1833 return;
1834 }
1835 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1836 try {
1837 setupWebRTC(media, iceServers, !includeCandidates);
1838 } catch (final WebRTCWrapper.InitializationException e) {
1839 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1840 webRTCWrapper.close();
1841 sendRetract(Reason.ofThrowable(e));
1842 return;
1843 }
1844 try {
1845 org.webrtc.SessionDescription webRTCSessionDescription =
1846 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1847 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1848 } catch (final Exception e) {
1849 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1850 // exceptions
1851 failureToInitiateSession(e, targetState);
1852 }
1853 }
1854
1855 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1856 if (isTerminated()) {
1857 return;
1858 }
1859 Log.d(
1860 Config.LOGTAG,
1861 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1862 Throwables.getRootCause(throwable));
1863 webRTCWrapper.close();
1864 final Reason reason = Reason.ofThrowable(throwable);
1865 if (isInState(targetState)) {
1866 sendSessionTerminate(reason, throwable.getMessage());
1867 } else {
1868 sendRetract(reason);
1869 }
1870 }
1871
1872 private void sendRetract(final Reason reason) {
1873 // TODO embed reason into retract
1874 sendJingleMessage("retract", id.with.asBareJid());
1875 transitionOrThrow(reasonToState(reason));
1876 this.finish();
1877 }
1878
1879 private void prepareSessionInitiate(
1880 final org.webrtc.SessionDescription webRTCSessionDescription,
1881 final boolean includeCandidates,
1882 final State targetState) {
1883 final SessionDescription sessionDescription =
1884 SessionDescription.parse(webRTCSessionDescription.description);
1885 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1886 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1887 if (includeCandidates) {
1888 candidates = parseCandidates(sessionDescription);
1889 } else {
1890 candidates = ImmutableMultimap.of();
1891 }
1892 this.initiatorRtpContentMap = rtpContentMap;
1893 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1894 encryptSessionInitiate(rtpContentMap);
1895 Futures.addCallback(
1896 outgoingContentMapFuture,
1897 new FutureCallback<>() {
1898 @Override
1899 public void onSuccess(final RtpContentMap outgoingContentMap) {
1900 if (includeCandidates) {
1901 Log.d(
1902 Config.LOGTAG,
1903 "including "
1904 + candidates.size()
1905 + " candidates in session initiate");
1906 sendSessionInitiate(
1907 outgoingContentMap.withCandidates(candidates), targetState);
1908 } else {
1909 sendSessionInitiate(outgoingContentMap, targetState);
1910 }
1911 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1912 }
1913
1914 @Override
1915 public void onFailure(@NonNull final Throwable throwable) {
1916 failureToInitiateSession(throwable, targetState);
1917 }
1918 },
1919 MoreExecutors.directExecutor());
1920 }
1921
1922 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1923 if (isTerminated()) {
1924 Log.w(
1925 Config.LOGTAG,
1926 id.account.getJid().asBareJid()
1927 + ": preparing session was too slow. already terminated. nothing to"
1928 + " do.");
1929 return;
1930 }
1931 this.transitionOrThrow(targetState);
1932 final Iq sessionInitiate =
1933 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1934 send(sessionInitiate);
1935 }
1936
1937 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1938 final RtpContentMap rtpContentMap) {
1939 if (this.omemoVerification.hasDeviceId()) {
1940 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1941 verifiedPayloadFuture =
1942 id.account
1943 .getAxolotlService()
1944 .encrypt(
1945 rtpContentMap,
1946 id.with,
1947 omemoVerification.getDeviceId());
1948 final ListenableFuture<RtpContentMap> future =
1949 Futures.transform(
1950 verifiedPayloadFuture,
1951 verifiedPayload -> {
1952 omemoVerification.setSessionFingerprint(
1953 verifiedPayload.getFingerprint());
1954 return verifiedPayload.getPayload();
1955 },
1956 MoreExecutors.directExecutor());
1957 if (Config.REQUIRE_RTP_VERIFICATION) {
1958 return future;
1959 }
1960 return Futures.catching(
1961 future,
1962 CryptoFailedException.class,
1963 e -> {
1964 Log.w(
1965 Config.LOGTAG,
1966 id.account.getJid().asBareJid()
1967 + ": unable to use OMEMO DTLS verification on outgoing"
1968 + " session initiate. falling back",
1969 e);
1970 return rtpContentMap;
1971 },
1972 MoreExecutors.directExecutor());
1973 } else {
1974 return Futures.immediateFuture(rtpContentMap);
1975 }
1976 }
1977
1978 protected void sendSessionTerminate(final Reason reason) {
1979 sendSessionTerminate(reason, null);
1980 }
1981
1982 protected void sendSessionTerminate(final Reason reason, final String text) {
1983 sendSessionTerminate(reason, text, this::writeLogMessage);
1984 sendJingleMessageFinish(reason);
1985 }
1986
1987 private void sendTransportInfo(
1988 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1989 final RtpContentMap transportInfo;
1990 try {
1991 final RtpContentMap rtpContentMap =
1992 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1993 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1994 } catch (final Exception e) {
1995 Log.d(
1996 Config.LOGTAG,
1997 id.account.getJid().asBareJid()
1998 + ": unable to prepare transport-info from candidate for content="
1999 + contentName);
2000 return;
2001 }
2002 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2003 send(iq);
2004 }
2005
2006 public RtpEndUserState getEndUserState() {
2007 switch (this.state) {
2008 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2009 if (isInitiator()) {
2010 return RtpEndUserState.RINGING;
2011 } else {
2012 return RtpEndUserState.INCOMING_CALL;
2013 }
2014 }
2015 case PROCEED -> {
2016 if (isInitiator()) {
2017 return RtpEndUserState.RINGING;
2018 } else {
2019 return RtpEndUserState.ACCEPTING_CALL;
2020 }
2021 }
2022 case SESSION_INITIALIZED_PRE_APPROVED -> {
2023 if (isInitiator()) {
2024 return RtpEndUserState.RINGING;
2025 } else {
2026 return RtpEndUserState.CONNECTING;
2027 }
2028 }
2029 case SESSION_ACCEPTED -> {
2030 final ContentAddition ca = getPendingContentAddition();
2031 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2032 return RtpEndUserState.INCOMING_CONTENT_ADD;
2033 }
2034 return getPeerConnectionStateAsEndUserState();
2035 }
2036 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2037 if (isInitiator()) {
2038 return RtpEndUserState.DECLINED_OR_BUSY;
2039 } else {
2040 return RtpEndUserState.ENDED;
2041 }
2042 }
2043 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2044 return RtpEndUserState.ENDED;
2045 }
2046 case RETRACTED_RACED -> {
2047 if (isInitiator()) {
2048 return RtpEndUserState.ENDED;
2049 } else {
2050 return RtpEndUserState.RETRACTED;
2051 }
2052 }
2053 case TERMINATED_CONNECTIVITY_ERROR -> {
2054 return zeroDuration()
2055 ? RtpEndUserState.CONNECTIVITY_ERROR
2056 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2057 }
2058 case TERMINATED_APPLICATION_FAILURE -> {
2059 return RtpEndUserState.APPLICATION_ERROR;
2060 }
2061 case TERMINATED_SECURITY_ERROR -> {
2062 return RtpEndUserState.SECURITY_ERROR;
2063 }
2064 }
2065 throw new IllegalStateException(
2066 String.format("%s has no equivalent EndUserState", this.state));
2067 }
2068
2069 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2070 final PeerConnection.PeerConnectionState state;
2071 try {
2072 state = webRTCWrapper.getState();
2073 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2074 // We usually close the WebRTCWrapper *before* transitioning so we might still
2075 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2076 return RtpEndUserState.ENDING_CALL;
2077 }
2078 return switch (state) {
2079 case CONNECTED -> RtpEndUserState.CONNECTED;
2080 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2081 case CLOSED -> RtpEndUserState.ENDING_CALL;
2082 default ->
2083 zeroDuration()
2084 ? RtpEndUserState.CONNECTIVITY_ERROR
2085 : RtpEndUserState.RECONNECTING;
2086 };
2087 }
2088
2089 private boolean isPeerConnectionConnected() {
2090 try {
2091 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2092 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2093 return false;
2094 }
2095 }
2096
2097 private void updateCallIntegrationState() {
2098 switch (this.state) {
2099 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2100 if (isInitiator()) {
2101 this.callIntegration.setDialing();
2102 } else {
2103 this.callIntegration.setRinging();
2104 }
2105 }
2106 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2107 if (isInitiator()) {
2108 this.callIntegration.setDialing();
2109 } else {
2110 this.callIntegration.setInitialized();
2111 }
2112 }
2113 case SESSION_ACCEPTED -> {
2114 if (isPeerConnectionConnected()) {
2115 this.callIntegration.setActive();
2116 } else {
2117 this.callIntegration.setInitialized();
2118 }
2119 }
2120 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2121 if (isInitiator()) {
2122 this.callIntegration.busy();
2123 } else {
2124 this.callIntegration.rejected();
2125 }
2126 }
2127 case TERMINATED_SUCCESS -> this.callIntegration.success();
2128 case ACCEPTED -> this.callIntegration.accepted();
2129 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT ->
2130 this.callIntegration.retracted();
2131 case TERMINATED_CONNECTIVITY_ERROR,
2132 TERMINATED_APPLICATION_FAILURE,
2133 TERMINATED_SECURITY_ERROR ->
2134 this.callIntegration.error();
2135 default ->
2136 throw new IllegalStateException(String.format("%s is not handled", this.state));
2137 }
2138 }
2139
2140 public ContentAddition getPendingContentAddition() {
2141 final RtpContentMap in = this.incomingContentAdd;
2142 final RtpContentMap out = this.outgoingContentAdd;
2143 if (out != null) {
2144 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2145 } else if (in != null) {
2146 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2147 } else {
2148 return null;
2149 }
2150 }
2151
2152 public Set<Media> getMedia() {
2153 final State current = getState();
2154 if (current == State.NULL) {
2155 if (isInitiator()) {
2156 return Preconditions.checkNotNull(
2157 this.proposedMedia, "RTP connection has not been initialized properly");
2158 }
2159 throw new IllegalStateException("RTP connection has not been initialized yet");
2160 }
2161 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2162 return Preconditions.checkNotNull(
2163 this.proposedMedia, "RTP connection has not been initialized properly");
2164 }
2165 final RtpContentMap localContentMap = getLocalContentMap();
2166 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2167 if (localContentMap != null) {
2168 return localContentMap.getMedia();
2169 } else if (initiatorContentMap != null) {
2170 return initiatorContentMap.getMedia();
2171 } else if (isTerminated()) {
2172 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2173 } else {
2174 return Preconditions.checkNotNull(
2175 this.proposedMedia, "RTP connection has not been initialized properly");
2176 }
2177 }
2178
2179 public boolean isVerified() {
2180 final String fingerprint = this.omemoVerification.getFingerprint();
2181 if (fingerprint == null) {
2182 return false;
2183 }
2184 final FingerprintStatus status =
2185 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2186 return status != null && status.isVerified();
2187 }
2188
2189 public boolean addMedia(final Media media) {
2190 final Set<Media> currentMedia = getMedia();
2191 if (currentMedia.contains(media)) {
2192 throw new IllegalStateException(String.format("%s has already been proposed", media));
2193 }
2194 // TODO add state protection - can only add while ACCEPTED or so
2195 Log.d(Config.LOGTAG, "adding media: " + media);
2196 return webRTCWrapper.addTrack(media);
2197 }
2198
2199 public synchronized void acceptCall() {
2200 switch (this.state) {
2201 case PROPOSED -> {
2202 cancelRingingTimeout();
2203 acceptCallFromProposed();
2204 }
2205 case SESSION_INITIALIZED -> {
2206 cancelRingingTimeout();
2207 acceptCallFromSessionInitialized();
2208 }
2209 case ACCEPTED ->
2210 Log.w(
2211 Config.LOGTAG,
2212 id.account.getJid().asBareJid()
2213 + ": the call has already been accepted with another client."
2214 + " UI was just lagging behind");
2215 case PROCEED, SESSION_ACCEPTED ->
2216 Log.w(
2217 Config.LOGTAG,
2218 id.account.getJid().asBareJid()
2219 + ": the call has already been accepted. user probably double"
2220 + " tapped the UI");
2221 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2222 }
2223 }
2224
2225 public synchronized void rejectCall() {
2226 if (isTerminated()) {
2227 Log.w(
2228 Config.LOGTAG,
2229 id.account.getJid().asBareJid()
2230 + ": received rejectCall() when session has already been terminated."
2231 + " nothing to do");
2232 return;
2233 }
2234 switch (this.state) {
2235 case PROPOSED -> rejectCallFromProposed();
2236 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2237 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2238 }
2239 }
2240
2241 public synchronized void integrationFailure() {
2242 final var state = getState();
2243 if (state == State.PROPOSED) {
2244 Log.e(
2245 Config.LOGTAG,
2246 id.account.getJid().asBareJid()
2247 + ": failed call integration in state proposed");
2248 rejectCallFromProposed();
2249 } else if (state == State.SESSION_INITIALIZED) {
2250 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2251 this.webRTCWrapper.close();
2252 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2253 } else {
2254 throw new IllegalStateException(
2255 String.format("Can not fail integration in state %s", state));
2256 }
2257 }
2258
2259 public synchronized void endCall() {
2260 if (isTerminated()) {
2261 Log.w(
2262 Config.LOGTAG,
2263 id.account.getJid().asBareJid()
2264 + ": received endCall() when session has already been terminated."
2265 + " nothing to do");
2266 return;
2267 }
2268 if (isInState(State.PROPOSED) && isResponder()) {
2269 rejectCallFromProposed();
2270 return;
2271 }
2272 if (isInState(State.PROCEED)) {
2273 if (isInitiator()) {
2274 retractFromProceed();
2275 } else {
2276 rejectCallFromProceed();
2277 }
2278 return;
2279 }
2280 if (isInitiator()
2281 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2282 this.webRTCWrapper.close();
2283 sendSessionTerminate(Reason.CANCEL);
2284 return;
2285 }
2286 if (isInState(State.SESSION_INITIALIZED)) {
2287 rejectCallFromSessionInitiate();
2288 return;
2289 }
2290 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2291 this.webRTCWrapper.close();
2292 sendSessionTerminate(Reason.SUCCESS);
2293 return;
2294 }
2295 if (isInState(
2296 State.TERMINATED_APPLICATION_FAILURE,
2297 State.TERMINATED_CONNECTIVITY_ERROR,
2298 State.TERMINATED_DECLINED_OR_BUSY)) {
2299 Log.d(
2300 Config.LOGTAG,
2301 "ignoring request to end call because already in state " + this.state);
2302 return;
2303 }
2304 throw new IllegalStateException(
2305 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2306 }
2307
2308 private void retractFromProceed() {
2309 Log.d(Config.LOGTAG, "retract from proceed");
2310 this.sendJingleMessage("retract");
2311 closeTransitionLogFinish(State.RETRACTED_RACED);
2312 }
2313
2314 private void closeTransitionLogFinish(final State state) {
2315 this.webRTCWrapper.close();
2316 transitionOrThrow(state);
2317 writeLogMessage(state);
2318 finish();
2319 }
2320
2321 private void setupWebRTC(
2322 final Set<Media> media,
2323 final List<PeerConnection.IceServer> iceServers,
2324 final boolean trickle)
2325 throws WebRTCWrapper.InitializationException {
2326 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2327 this.webRTCWrapper.setup(this.xmppConnectionService);
2328 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2329 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2330 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2331 }
2332
2333 private void acceptCallFromProposed() {
2334 transitionOrThrow(State.PROCEED);
2335 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2336 this.callIntegration.startAudioRouting();
2337 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2338 this.sendJingleMessage("proceed");
2339 }
2340
2341 private void rejectCallFromProposed() {
2342 transitionOrThrow(State.REJECTED);
2343 writeLogMessageMissed();
2344 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2345 this.sendJingleMessage("reject");
2346 finish();
2347 }
2348
2349 private void rejectCallFromProceed() {
2350 this.sendJingleMessage("reject");
2351 closeTransitionLogFinish(State.REJECTED_RACED);
2352 }
2353
2354 private void rejectCallFromSessionInitiate() {
2355 webRTCWrapper.close();
2356 sendSessionTerminate(Reason.DECLINE);
2357 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2358 }
2359
2360 private void sendJingleMessage(final String action) {
2361 sendJingleMessage(action, id.with);
2362 }
2363
2364 private void sendJingleMessage(final String action, final Jid to) {
2365 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2366 messagePacket.setType(
2367 im.conversations.android.xmpp.model.stanza.Message.Type
2368 .CHAT); // we want to carbon copy those
2369 messagePacket.setTo(to);
2370 final Element intent =
2371 messagePacket
2372 .addChild(action, Namespace.JINGLE_MESSAGE)
2373 .setAttribute("id", id.sessionId);
2374 if ("proceed".equals(action)) {
2375 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2376 if (isOmemoEnabled()) {
2377 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2378 final Element device =
2379 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2380 device.setAttribute("id", deviceId);
2381 }
2382 }
2383 messagePacket.addChild("store", "urn:xmpp:hints");
2384 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2385 }
2386
2387 private void sendJingleMessageFinish(final Reason reason) {
2388 final var account = id.getAccount();
2389 final var messagePacket =
2390 xmppConnectionService
2391 .getMessageGenerator()
2392 .sessionFinish(id.with, id.sessionId, reason);
2393 xmppConnectionService.sendMessagePacket(account, messagePacket);
2394 }
2395
2396 private boolean isOmemoEnabled() {
2397 final Conversational conversational = message.getConversation();
2398 if (conversational instanceof Conversation) {
2399 return ((Conversation) conversational).getNextEncryption()
2400 == Message.ENCRYPTION_AXOLOTL;
2401 }
2402 return false;
2403 }
2404
2405 private void acceptCallFromSessionInitialized() {
2406 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2407 this.callIntegration.startAudioRouting();
2408 sendSessionAccept();
2409 }
2410
2411 @Override
2412 protected synchronized boolean transition(final State target, final Runnable runnable) {
2413 if (super.transition(target, runnable)) {
2414 updateEndUserState();
2415 updateOngoingCallNotification();
2416 return true;
2417 } else {
2418 return false;
2419 }
2420 }
2421
2422 @Override
2423 public void onIceCandidate(final IceCandidate iceCandidate) {
2424 final RtpContentMap rtpContentMap =
2425 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2426 final IceUdpTransportInfo.Credentials credentials;
2427 try {
2428 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2429 } catch (final IllegalArgumentException e) {
2430 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2431 return;
2432 }
2433 final String uFrag = credentials.ufrag;
2434 final IceUdpTransportInfo.Candidate candidate =
2435 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2436 if (candidate == null) {
2437 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2438 return;
2439 }
2440 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2441 sendTransportInfo(iceCandidate.sdpMid, candidate);
2442 }
2443
2444 @Override
2445 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2446 Log.d(
2447 Config.LOGTAG,
2448 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2449 this.stateHistory.add(newState);
2450 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2451 this.sessionDuration.start();
2452 updateOngoingCallNotification();
2453 } else if (this.sessionDuration.isRunning()) {
2454 this.sessionDuration.stop();
2455 updateOngoingCallNotification();
2456 }
2457
2458 final boolean neverConnected =
2459 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2460
2461 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2462 if (neverConnected) {
2463 if (isTerminated()) {
2464 Log.d(
2465 Config.LOGTAG,
2466 id.account.getJid().asBareJid()
2467 + ": not sending session-terminate after connectivity error"
2468 + " because session is already in state "
2469 + this.state);
2470 return;
2471 }
2472 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2473 return;
2474 } else {
2475 this.restartIce();
2476 }
2477 }
2478 updateEndUserState();
2479 }
2480
2481 private void restartIce() {
2482 this.stateHistory.clear();
2483 this.webRTCWrapper.restartIceAsync();
2484 }
2485
2486 @Override
2487 public void onRenegotiationNeeded() {
2488 this.webRTCWrapper.execute(this::renegotiate);
2489 }
2490
2491 private synchronized void renegotiate() {
2492 final SessionDescription sessionDescription;
2493 try {
2494 sessionDescription = setLocalSessionDescription();
2495 } catch (final Exception e) {
2496 final Throwable cause = Throwables.getRootCause(e);
2497 webRTCWrapper.close();
2498 if (isTerminated()) {
2499 Log.d(
2500 Config.LOGTAG,
2501 "failed to renegotiate. session was already terminated",
2502 cause);
2503 return;
2504 }
2505 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2506 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2507 return;
2508 }
2509 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2510 final RtpContentMap currentContentMap = getLocalContentMap();
2511 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2512 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2513
2514 Log.d(
2515 Config.LOGTAG,
2516 id.getAccount().getJid().asBareJid()
2517 + ": renegotiate. iceRestart="
2518 + iceRestart
2519 + " content id diff="
2520 + diff);
2521
2522 if (diff.hasModifications() && iceRestart) {
2523 webRTCWrapper.close();
2524 sendSessionTerminate(
2525 Reason.FAILED_APPLICATION,
2526 "WebRTC unexpectedly tried to modify content and transport at once");
2527 return;
2528 }
2529
2530 if (iceRestart) {
2531 initiateIceRestart(rtpContentMap);
2532 return;
2533 } else if (diff.isEmpty()) {
2534 Log.d(
2535 Config.LOGTAG,
2536 "renegotiation. nothing to do. SignalingState="
2537 + this.webRTCWrapper.getSignalingState());
2538 }
2539
2540 if (diff.added.isEmpty()) {
2541 return;
2542 }
2543 modifyLocalContentMap(rtpContentMap);
2544 sendContentAdd(rtpContentMap, diff.added);
2545 }
2546
2547 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2548 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2549 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2550 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2551 iq.setTo(id.with);
2552 xmppConnectionService.sendIqPacket(
2553 id.account,
2554 iq,
2555 (response) -> {
2556 if (response.getType() == Iq.Type.RESULT) {
2557 Log.d(Config.LOGTAG, "received success to our ice restart");
2558 setLocalContentMap(rtpContentMap);
2559 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2560 return;
2561 }
2562 if (response.getType() == Iq.Type.ERROR) {
2563 if (isTieBreak(response)) {
2564 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2565 return;
2566 }
2567 handleIqErrorResponse(response);
2568 }
2569 if (response.getType() == Iq.Type.TIMEOUT) {
2570 handleIqTimeoutResponse(response);
2571 }
2572 });
2573 }
2574
2575 private boolean isTieBreak(final Iq response) {
2576 final Element error = response.findChild("error");
2577 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2578 }
2579
2580 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2581 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2582 this.outgoingContentAdd = contentAdd;
2583 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2584 prepareOutgoingContentMap(contentAdd);
2585 Futures.addCallback(
2586 outgoingContentMapFuture,
2587 new FutureCallback<>() {
2588 @Override
2589 public void onSuccess(final RtpContentMap outgoingContentMap) {
2590 sendContentAdd(outgoingContentMap);
2591 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2592 }
2593
2594 @Override
2595 public void onFailure(@NonNull Throwable throwable) {
2596 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2597 }
2598 },
2599 MoreExecutors.directExecutor());
2600 }
2601
2602 private void sendContentAdd(final RtpContentMap contentAdd) {
2603
2604 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2605 iq.setTo(id.with);
2606 xmppConnectionService.sendIqPacket(
2607 id.account,
2608 iq,
2609 (response) -> {
2610 if (response.getType() == Iq.Type.RESULT) {
2611 Log.d(
2612 Config.LOGTAG,
2613 id.getAccount().getJid().asBareJid()
2614 + ": received ACK to our content-add");
2615 return;
2616 }
2617 if (response.getType() == Iq.Type.ERROR) {
2618 if (isTieBreak(response)) {
2619 this.outgoingContentAdd = null;
2620 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2621 return;
2622 }
2623 handleIqErrorResponse(response);
2624 }
2625 if (response.getType() == Iq.Type.TIMEOUT) {
2626 handleIqTimeoutResponse(response);
2627 }
2628 });
2629 }
2630
2631 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2632 if (isInitiator()) {
2633 this.initiatorRtpContentMap = rtpContentMap;
2634 } else {
2635 this.responderRtpContentMap = rtpContentMap;
2636 }
2637 }
2638
2639 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2640 if (isInitiator()) {
2641 this.responderRtpContentMap = rtpContentMap;
2642 } else {
2643 this.initiatorRtpContentMap = rtpContentMap;
2644 }
2645 }
2646
2647 // this method is to be used for content map modifications that modify media
2648 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2649 final RtpContentMap activeContents = rtpContentMap.activeContents();
2650 setLocalContentMap(activeContents);
2651 this.callIntegration.setAudioDeviceWhenAvailable(
2652 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2653 updateEndUserState();
2654 }
2655
2656 private SessionDescription setLocalSessionDescription()
2657 throws ExecutionException, InterruptedException {
2658 final org.webrtc.SessionDescription sessionDescription =
2659 this.webRTCWrapper.setLocalDescription(false).get();
2660 return SessionDescription.parse(sessionDescription.description);
2661 }
2662
2663 private void closeWebRTCSessionAfterFailedConnection() {
2664 this.webRTCWrapper.close();
2665 synchronized (this) {
2666 if (isTerminated()) {
2667 Log.d(
2668 Config.LOGTAG,
2669 id.account.getJid().asBareJid()
2670 + ": no need to send session-terminate after failed connection."
2671 + " Other party already did");
2672 return;
2673 }
2674 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2675 }
2676 }
2677
2678 public boolean zeroDuration() {
2679 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2680 }
2681
2682 public long getCallDuration() {
2683 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2684 }
2685
2686 @Override
2687 public CallIntegration getCallIntegration() {
2688 return this.callIntegration;
2689 }
2690
2691 public boolean isMicrophoneEnabled() {
2692 return webRTCWrapper.isMicrophoneEnabled();
2693 }
2694
2695 public boolean setMicrophoneEnabled(final boolean enabled) {
2696 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2697 }
2698
2699 public boolean isVideoEnabled() {
2700 return webRTCWrapper.isVideoEnabled();
2701 }
2702
2703 public void setVideoEnabled(final boolean enabled) {
2704 webRTCWrapper.setVideoEnabled(enabled);
2705 }
2706
2707 public boolean isCameraSwitchable() {
2708 return webRTCWrapper.isCameraSwitchable();
2709 }
2710
2711 public boolean isFrontCamera() {
2712 return webRTCWrapper.isFrontCamera();
2713 }
2714
2715 public ListenableFuture<Boolean> switchCamera() {
2716 return webRTCWrapper.switchCamera();
2717 }
2718
2719 @Override
2720 public synchronized void onCallIntegrationShowIncomingCallUi() {
2721 if (isTerminated()) {
2722 // there might be race conditions with the call integration service invoking this
2723 // callback when the rtp session has already ended.
2724 Log.w(
2725 Config.LOGTAG,
2726 "CallIntegration requested incoming call UI but session was already"
2727 + " terminated");
2728 return;
2729 }
2730 // TODO apparently this can be called too early as well?
2731 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2732 }
2733
2734 @Override
2735 public void onCallIntegrationDisconnect() {
2736 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2737 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2738 rejectCall();
2739 } else {
2740 endCall();
2741 }
2742 }
2743
2744 @Override
2745 public void onCallIntegrationReject() {
2746 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2747 try {
2748 rejectCall();
2749 } catch (final IllegalStateException e) {
2750 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2751 }
2752 }
2753
2754 @Override
2755 public void onCallIntegrationAnswer() {
2756 // we need to start the UI to a) show it and b) be able to ask for permissions
2757 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2758 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2759 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toString());
2760 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString());
2761 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2762 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2763 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2764 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2765 xmppConnectionService.startActivity(intent);
2766 }
2767
2768 @Override
2769 public void onCallIntegrationSilence() {
2770 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2771 }
2772
2773 @Override
2774 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2775 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2776 // that was previous disabled by the user. A proper implementation would probably be to
2777 // track user choice and enable the microphone with a userEnabled() &&
2778 // callIntegration.isMicrophoneEnabled() condition
2779 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2780 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2781 }
2782
2783 @Override
2784 public void onAudioDeviceChanged(
2785 final CallIntegration.AudioDevice selectedAudioDevice,
2786 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2787 Log.d(
2788 Config.LOGTAG,
2789 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2790 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2791 selectedAudioDevice, availableAudioDevices);
2792 }
2793
2794 private void updateEndUserState() {
2795 final RtpEndUserState endUserState = getEndUserState();
2796 this.updateCallIntegrationState();
2797 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2798 id.account, id.with, id.sessionId, endUserState);
2799 }
2800
2801 private void updateOngoingCallNotification() {
2802 final State state = this.state;
2803 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2804 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2805 && isResponder()) {
2806 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2807 xmppConnectionService.removeOngoingCall();
2808 return;
2809 }
2810 final boolean reconnecting;
2811 if (state == State.SESSION_ACCEPTED) {
2812 reconnecting =
2813 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2814 } else {
2815 reconnecting = false;
2816 }
2817 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2818 } else {
2819 xmppConnectionService.removeOngoingCall();
2820 }
2821 }
2822
2823 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2824 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2825 final Iq request = new Iq(Iq.Type.GET);
2826 request.setTo(id.account.getDomain());
2827 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2828 xmppConnectionService.sendIqPacket(
2829 id.account,
2830 request,
2831 (response) -> {
2832 final var iceServers = IceServers.parse(response);
2833 if (iceServers.isEmpty()) {
2834 Log.w(
2835 Config.LOGTAG,
2836 id.account.getJid().asBareJid()
2837 + ": no ICE server found "
2838 + response);
2839 }
2840 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2841 });
2842 } else {
2843 Log.w(
2844 Config.LOGTAG,
2845 id.account.getJid().asBareJid() + ": has no external service discovery");
2846 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2847 }
2848 }
2849
2850 @Override
2851 protected void terminateTransport() {
2852 this.webRTCWrapper.close();
2853 }
2854
2855 @Override
2856 protected void finish() {
2857 if (isTerminated()) {
2858 this.cancelRingingTimeout();
2859 this.callIntegration.verifyDisconnected();
2860 this.webRTCWrapper.verifyClosed();
2861 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2862 super.finish();
2863 } else {
2864 throw new IllegalStateException(
2865 String.format("Unable to call finish from %s", this.state));
2866 }
2867 }
2868
2869 private void writeLogMessage(final State state) {
2870 final long duration = getCallDuration();
2871 if (state == State.TERMINATED_SUCCESS
2872 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2873 writeLogMessageSuccess(duration);
2874 } else {
2875 writeLogMessageMissed();
2876 }
2877 }
2878
2879 private void writeLogMessageSuccess(final long duration) {
2880 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2881 this.writeMessage();
2882 }
2883
2884 private void writeLogMessageMissed() {
2885 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2886 this.writeMessage();
2887 }
2888
2889 private void writeMessage() {
2890 final Conversational conversational = message.getConversation();
2891 if (conversational instanceof Conversation) {
2892 ((Conversation) conversational).add(this.message);
2893 xmppConnectionService.createMessageAsync(message);
2894 xmppConnectionService.updateConversationUi();
2895 } else {
2896 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2897 }
2898 }
2899
2900 public Optional<VideoTrack> getLocalVideoTrack() {
2901 return webRTCWrapper.getLocalVideoTrack();
2902 }
2903
2904 public Optional<VideoTrack> getRemoteVideoTrack() {
2905 return webRTCWrapper.getRemoteVideoTrack();
2906 }
2907
2908 public EglBase.Context getEglBaseContext() {
2909 return webRTCWrapper.getEglBaseContext();
2910 }
2911
2912 void setProposedMedia(final Set<Media> media) {
2913 this.proposedMedia = media;
2914 this.callIntegration.setVideoState(
2915 Media.audioOnly(media)
2916 ? VideoProfile.STATE_AUDIO_ONLY
2917 : VideoProfile.STATE_BIDIRECTIONAL);
2918 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2919 }
2920
2921 public void fireStateUpdate() {
2922 final RtpEndUserState endUserState = getEndUserState();
2923 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2924 id.account, id.with, id.sessionId, endUserState);
2925 }
2926
2927 public boolean isSwitchToVideoAvailable() {
2928 final boolean prerequisite =
2929 Media.audioOnly(getMedia())
2930 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2931 .contains(getEndUserState());
2932 return prerequisite && remoteHasVideoFeature();
2933 }
2934
2935 private boolean remoteHasVideoFeature() {
2936 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2937 }
2938
2939 private boolean remoteHasSdpOfferAnswer() {
2940 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2941 }
2942
2943 @Override
2944 public Account getAccount() {
2945 return id.account;
2946 }
2947
2948 @Override
2949 public Jid getWith() {
2950 return id.with;
2951 }
2952
2953 @Override
2954 public String getSessionId() {
2955 return id.sessionId;
2956 }
2957
2958 private interface OnIceServersDiscovered {
2959 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2960 }
2961}