1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import androidx.annotation.NonNull;
8import androidx.annotation.Nullable;
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableMultimap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Iterables;
19import com.google.common.collect.Maps;
20import com.google.common.collect.Sets;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25import eu.siacs.conversations.BuildConfig;
26import eu.siacs.conversations.Config;
27import eu.siacs.conversations.crypto.axolotl.AxolotlService;
28import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
29import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
30import eu.siacs.conversations.entities.Account;
31import eu.siacs.conversations.entities.Conversation;
32import eu.siacs.conversations.entities.Conversational;
33import eu.siacs.conversations.entities.Message;
34import eu.siacs.conversations.entities.RtpSessionStatus;
35import eu.siacs.conversations.services.CallIntegration;
36import eu.siacs.conversations.ui.RtpSessionActivity;
37import eu.siacs.conversations.xml.Element;
38import eu.siacs.conversations.xml.Namespace;
39import eu.siacs.conversations.xmpp.Jid;
40import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
41import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
42import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
46import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
47import im.conversations.android.xmpp.model.disco.external.Services;
48import im.conversations.android.xmpp.model.jingle.Jingle;
49import im.conversations.android.xmpp.model.stanza.Iq;
50import java.util.Arrays;
51import java.util.Collection;
52import java.util.Collections;
53import java.util.LinkedList;
54import java.util.List;
55import java.util.Map;
56import java.util.Queue;
57import java.util.Set;
58import java.util.concurrent.ExecutionException;
59import java.util.concurrent.ScheduledFuture;
60import java.util.concurrent.TimeUnit;
61import org.webrtc.EglBase;
62import org.webrtc.IceCandidate;
63import org.webrtc.PeerConnection;
64import org.webrtc.VideoTrack;
65
66public class JingleRtpConnection extends AbstractJingleConnection
67 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
68
69 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
70 public static final List<State> STATES_SHOWING_ONGOING_CALL =
71 Arrays.asList(
72 State.PROPOSED,
73 State.PROCEED,
74 State.SESSION_INITIALIZED_PRE_APPROVED,
75 State.SESSION_ACCEPTED);
76 private static final long BUSY_TIME_OUT = 30;
77
78 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
79 private final Queue<
80 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
81 pendingIceCandidates = new LinkedList<>();
82 private final OmemoVerification omemoVerification = new OmemoVerification();
83 private final CallIntegration callIntegration;
84 private final Message message;
85
86 private Set<Media> proposedMedia;
87 private RtpContentMap initiatorRtpContentMap;
88 private RtpContentMap responderRtpContentMap;
89 private RtpContentMap incomingContentAdd;
90 private RtpContentMap outgoingContentAdd;
91 private IceUdpTransportInfo.Setup peerDtlsSetup;
92 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
93 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
94 private ScheduledFuture<?> ringingTimeoutFuture;
95
96 JingleRtpConnection(
97 final JingleConnectionManager jingleConnectionManager,
98 final Id id,
99 final Jid initiator) {
100 this(
101 jingleConnectionManager,
102 id,
103 initiator,
104 new CallIntegration(
105 jingleConnectionManager
106 .getXmppConnectionService()
107 .getApplicationContext()));
108 this.callIntegration.setAddress(
109 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
110 final var contact = id.getContact();
111 this.callIntegration.setCallerDisplayName(
112 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
113 this.callIntegration.setInitialized();
114 }
115
116 JingleRtpConnection(
117 final JingleConnectionManager jingleConnectionManager,
118 final Id id,
119 final Jid initiator,
120 final CallIntegration callIntegration) {
121 super(jingleConnectionManager, id, initiator);
122 final Conversation conversation =
123 jingleConnectionManager
124 .getXmppConnectionService()
125 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
126 this.message =
127 new Message(
128 conversation,
129 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
130 Message.TYPE_RTP_SESSION,
131 id.sessionId);
132 this.callIntegration = callIntegration;
133 this.callIntegration.setCallback(this);
134 }
135
136 @Override
137 synchronized void deliverPacket(final Iq iq) {
138 final var jingle = iq.getExtension(Jingle.class);
139 switch (jingle.getAction()) {
140 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
141 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
142 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
143 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
144 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
145 case CONTENT_ACCEPT -> receiveContentAccept(iq);
146 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
147 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
148 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
149 default -> {
150 respondOk(iq);
151 Log.d(
152 Config.LOGTAG,
153 String.format(
154 "%s: received unhandled jingle action %s",
155 id.account.getJid().asBareJid(), jingle.getAction()));
156 }
157 }
158 }
159
160 @Override
161 synchronized void notifyRebound() {
162 if (isTerminated()) {
163 return;
164 }
165 webRTCWrapper.close();
166 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
167 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
168 }
169 if (isInState(
170 State.SESSION_INITIALIZED,
171 State.SESSION_INITIALIZED_PRE_APPROVED,
172 State.SESSION_ACCEPTED)) {
173 // we might have already changed resources (full jid) at this point; so this might not
174 // even reach the other party
175 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
176 } else {
177 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
178 finish();
179 }
180 }
181
182 private void receiveSessionTerminate(final Iq jinglePacket) {
183 respondOk(jinglePacket);
184 final var jingle = jinglePacket.getExtension(Jingle.class);
185 final Jingle.ReasonWrapper wrapper = jingle.getReason();
186 final State previous = this.state;
187 Log.d(
188 Config.LOGTAG,
189 id.account.getJid().asBareJid()
190 + ": received session terminate reason="
191 + wrapper.reason
192 + "("
193 + Strings.nullToEmpty(wrapper.text)
194 + ") while in state "
195 + previous);
196 if (TERMINATED.contains(previous)) {
197 Log.d(
198 Config.LOGTAG,
199 id.account.getJid().asBareJid()
200 + ": ignoring session terminate because already in "
201 + previous);
202 return;
203 }
204 webRTCWrapper.close();
205 final State target = reasonToState(wrapper.reason);
206 transitionOrThrow(target);
207 writeLogMessage(target);
208 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
209 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
210 }
211 finish();
212 }
213
214 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
215 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
216 // INITIALIZED only after transport-info has been received
217 if (isInState(
218 State.NULL,
219 State.PROCEED,
220 State.SESSION_INITIALIZED,
221 State.SESSION_INITIALIZED_PRE_APPROVED,
222 State.SESSION_ACCEPTED)) {
223 final RtpContentMap contentMap;
224 try {
225 contentMap = RtpContentMap.of(jingle);
226 } catch (final IllegalArgumentException | NullPointerException e) {
227 Log.d(
228 Config.LOGTAG,
229 id.account.getJid().asBareJid()
230 + ": improperly formatted contents; ignoring",
231 e);
232 respondOk(jinglePacket);
233 return;
234 }
235 receiveTransportInfo(jinglePacket, contentMap);
236 } else {
237 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
238 }
239 }
240
241 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
242 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
243 candidates = contentMap.contents.entrySet();
244 final RtpContentMap remote = getRemoteContentMap();
245 final Set<String> remoteContentIds =
246 remote == null ? Collections.emptySet() : remote.contents.keySet();
247 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
248 Log.d(
249 Config.LOGTAG,
250 "received transport-info for unknown contents "
251 + contentMap.contents.keySet()
252 + " (known: "
253 + remoteContentIds
254 + ")");
255 respondOk(jinglePacket);
256 pendingIceCandidates.addAll(candidates);
257 return;
258 }
259 if (this.state != State.SESSION_ACCEPTED) {
260 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
261 respondOk(jinglePacket);
262 pendingIceCandidates.addAll(candidates);
263 return;
264 }
265 // zero candidates + modified credentials are an ICE restart offer
266 if (checkForIceRestart(jinglePacket, contentMap)) {
267 return;
268 }
269 respondOk(jinglePacket);
270 try {
271 processCandidates(candidates);
272 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
273 Log.w(
274 Config.LOGTAG,
275 id.account.getJid().asBareJid()
276 + ": PeerConnection was not initialized when processing transport info."
277 + " this usually indicates a race condition that can be ignored");
278 }
279 }
280
281 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
282 final RtpContentMap modification;
283 try {
284 modification = RtpContentMap.of(jingle);
285 modification.requireContentDescriptions();
286 } catch (final RuntimeException e) {
287 Log.d(
288 Config.LOGTAG,
289 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
290 Throwables.getRootCause(e));
291 respondOk(iq);
292 webRTCWrapper.close();
293 sendSessionTerminate(Reason.of(e), e.getMessage());
294 return;
295 }
296 if (isInState(State.SESSION_ACCEPTED)) {
297 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
298 final ListenableFuture<RtpContentMap> future =
299 receiveRtpContentMap(
300 modification,
301 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
302 Futures.addCallback(
303 future,
304 new FutureCallback<>() {
305 @Override
306 public void onSuccess(final RtpContentMap rtpContentMap) {
307 receiveContentAdd(iq, rtpContentMap);
308 }
309
310 @Override
311 public void onFailure(@NonNull Throwable throwable) {
312 respondOk(iq);
313 final Throwable rootCause = Throwables.getRootCause(throwable);
314 Log.d(
315 Config.LOGTAG,
316 id.account.getJid().asBareJid()
317 + ": improperly formatted contents in content-add",
318 throwable);
319 webRTCWrapper.close();
320 sendSessionTerminate(
321 Reason.ofThrowable(rootCause), rootCause.getMessage());
322 }
323 },
324 MoreExecutors.directExecutor());
325 } else {
326 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
327 }
328 }
329
330 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
331 final RtpContentMap remote = getRemoteContentMap();
332 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
333 respondOk(jinglePacket);
334 this.webRTCWrapper.close();
335 sendSessionTerminate(
336 Reason.FAILED_APPLICATION,
337 String.format(
338 "contents with names %s already exists",
339 Joiner.on(", ").join(modification.getNames())));
340 return;
341 }
342 final ContentAddition contentAddition =
343 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
344
345 final RtpContentMap outgoing = this.outgoingContentAdd;
346 final Set<ContentAddition.Summary> outgoingContentAddSummary =
347 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
348
349 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
350 if (isInitiator()) {
351 Log.d(
352 Config.LOGTAG,
353 id.getAccount().getJid().asBareJid()
354 + ": respond with tie break to matching content-add offer");
355 respondWithTieBreak(jinglePacket);
356 } else {
357 Log.d(
358 Config.LOGTAG,
359 id.getAccount().getJid().asBareJid()
360 + ": automatically accept matching content-add offer");
361 acceptContentAdd(contentAddition.summary, modification);
362 }
363 return;
364 }
365
366 // once we can display multiple video tracks we can be more loose with this condition
367 // theoretically it should also be fine to automatically accept audio only contents
368 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
369 Log.d(
370 Config.LOGTAG,
371 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
372 this.incomingContentAdd = modification;
373 respondOk(jinglePacket);
374 updateEndUserState();
375 } else {
376 respondOk(jinglePacket);
377 // TODO do we want to add a reason?
378 rejectContentAdd(modification);
379 }
380 }
381
382 private void receiveContentAccept(final Iq jinglePacket) {
383 final var jingle = jinglePacket.getExtension(Jingle.class);
384 final RtpContentMap receivedContentAccept;
385 try {
386 receivedContentAccept = RtpContentMap.of(jingle);
387 receivedContentAccept.requireContentDescriptions();
388 } catch (final RuntimeException e) {
389 Log.d(
390 Config.LOGTAG,
391 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
392 Throwables.getRootCause(e));
393 respondOk(jinglePacket);
394 webRTCWrapper.close();
395 sendSessionTerminate(Reason.of(e), e.getMessage());
396 return;
397 }
398
399 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
400 if (outgoingContentAdd == null) {
401 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
402 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
403 return;
404 }
405 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
406 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
407 this.outgoingContentAdd = null;
408 respondOk(jinglePacket);
409 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
410 final ListenableFuture<RtpContentMap> future =
411 receiveRtpContentMap(
412 receivedContentAccept,
413 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
414 Futures.addCallback(
415 future,
416 new FutureCallback<>() {
417 @Override
418 public void onSuccess(final RtpContentMap result) {
419 receiveContentAccept(result);
420 }
421
422 @Override
423 public void onFailure(@NonNull final Throwable throwable) {
424 webRTCWrapper.close();
425 sendSessionTerminate(
426 Reason.ofThrowable(throwable), throwable.getMessage());
427 }
428 },
429 MoreExecutors.directExecutor());
430 } else {
431 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
432 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
433 }
434 }
435
436 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
437 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
438 final RtpContentMap modifiedContentMap =
439 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
440
441 setRemoteContentMap(modifiedContentMap);
442
443 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
444
445 final org.webrtc.SessionDescription sdp =
446 new org.webrtc.SessionDescription(
447 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
448
449 try {
450 this.webRTCWrapper.setRemoteDescription(sdp).get();
451 } catch (final Exception e) {
452 final Throwable cause = Throwables.getRootCause(e);
453 Log.d(
454 Config.LOGTAG,
455 id.getAccount().getJid().asBareJid()
456 + ": unable to set remote description after receiving content-accept",
457 cause);
458 webRTCWrapper.close();
459 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
460 return;
461 }
462 Log.d(
463 Config.LOGTAG,
464 id.getAccount().getJid().asBareJid()
465 + ": remote has accepted content-add "
466 + ContentAddition.summary(receivedContentAccept));
467 processCandidates(receivedContentAccept.contents.entrySet());
468 updateEndUserState();
469 }
470
471 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
472 if (this.state != State.SESSION_ACCEPTED) {
473 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
474 return;
475 }
476 final Map<String, Content.Senders> modification =
477 Maps.transformEntries(
478 jingle.getJingleContents(), (key, value) -> value.getSenders());
479 final boolean isInitiator = isInitiator();
480 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
481 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
482 final Set<String> currentOutgoingMediaIds =
483 currentOutgoing == null
484 ? Collections.emptySet()
485 : currentOutgoing.contents.keySet();
486 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
487 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
488 respondOk(jinglePacket);
489 final RtpContentMap modifiedContentMap;
490 try {
491 modifiedContentMap =
492 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
493 } catch (final IllegalArgumentException e) {
494 webRTCWrapper.close();
495 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
496 return;
497 }
498 this.outgoingContentAdd = modifiedContentMap;
499 Log.d(
500 Config.LOGTAG,
501 id.account.getJid().asBareJid()
502 + ": processed content-modification for pending content-add");
503 } else if (remoteContentMap != null
504 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
505 respondOk(jinglePacket);
506 final RtpContentMap modifiedRemoteContentMap;
507 try {
508 modifiedRemoteContentMap =
509 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
510 } catch (final IllegalArgumentException e) {
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
513 return;
514 }
515 final SessionDescription offer;
516 try {
517 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
518 } catch (final IllegalArgumentException | NullPointerException e) {
519 Log.d(
520 Config.LOGTAG,
521 id.getAccount().getJid().asBareJid()
522 + ": unable convert offer from content-modify to SDP",
523 e);
524 webRTCWrapper.close();
525 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
526 return;
527 }
528 Log.d(
529 Config.LOGTAG,
530 id.account.getJid().asBareJid() + ": auto accepting content-modification");
531 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
532 } else {
533 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
534 respondWithItemNotFound(jinglePacket);
535 }
536 }
537
538 private void autoAcceptContentModify(
539 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
540 this.setRemoteContentMap(modifiedRemoteContentMap);
541 final org.webrtc.SessionDescription sdp =
542 new org.webrtc.SessionDescription(
543 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
544 try {
545 this.webRTCWrapper.setRemoteDescription(sdp).get();
546 // auto accept is only done when we already have tracks
547 final SessionDescription answer = setLocalSessionDescription();
548 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
549 modifyLocalContentMap(rtpContentMap);
550 // we do not need to send an answer but do we have to resend the candidates currently in
551 // SDP?
552 // resendCandidatesFromSdp(answer);
553 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
554 } catch (final Exception e) {
555 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
556 webRTCWrapper.close();
557 sendSessionTerminate(Reason.FAILED_APPLICATION);
558 }
559 }
560
561 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
562 final SessionDescription answer) {
563 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
564 new ImmutableMultimap.Builder<>();
565 for (final SessionDescription.Media media : answer.media) {
566 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
567 if (Strings.isNullOrEmpty(mid)) {
568 continue;
569 }
570 for (final String sdpCandidate : media.attributes.get("candidate")) {
571 final IceUdpTransportInfo.Candidate candidate =
572 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
573 if (candidate != null) {
574 candidateBuilder.put(mid, candidate);
575 }
576 }
577 }
578 return candidateBuilder.build();
579 }
580
581 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
582 final RtpContentMap receivedContentReject;
583 try {
584 receivedContentReject = RtpContentMap.of(jingle);
585 } catch (final RuntimeException e) {
586 Log.d(
587 Config.LOGTAG,
588 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
589 Throwables.getRootCause(e));
590 respondOk(jinglePacket);
591 this.webRTCWrapper.close();
592 sendSessionTerminate(Reason.of(e), e.getMessage());
593 return;
594 }
595
596 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
597 if (outgoingContentAdd == null) {
598 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
599 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
600 return;
601 }
602 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
603 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
604 this.outgoingContentAdd = null;
605 respondOk(jinglePacket);
606 Log.d(Config.LOGTAG, jinglePacket.toString());
607 receiveContentReject(ourSummary);
608 } else {
609 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
610 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
611 }
612 }
613
614 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
615 try {
616 this.webRTCWrapper.removeTrack(Media.VIDEO);
617 final RtpContentMap localContentMap = customRollback();
618 modifyLocalContentMap(localContentMap);
619 } catch (final Exception e) {
620 final Throwable cause = Throwables.getRootCause(e);
621 Log.d(
622 Config.LOGTAG,
623 id.getAccount().getJid().asBareJid()
624 + ": unable to rollback local description after receiving"
625 + " content-reject",
626 cause);
627 webRTCWrapper.close();
628 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
629 return;
630 }
631 Log.d(
632 Config.LOGTAG,
633 id.getAccount().getJid().asBareJid()
634 + ": remote has rejected our content-add "
635 + summary);
636 }
637
638 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
639 final RtpContentMap receivedContentRemove;
640 try {
641 receivedContentRemove = RtpContentMap.of(jingle);
642 receivedContentRemove.requireContentDescriptions();
643 } catch (final RuntimeException e) {
644 Log.d(
645 Config.LOGTAG,
646 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
647 Throwables.getRootCause(e));
648 respondOk(jinglePacket);
649 this.webRTCWrapper.close();
650 sendSessionTerminate(Reason.of(e), e.getMessage());
651 return;
652 }
653 respondOk(jinglePacket);
654 receiveContentRemove(receivedContentRemove);
655 }
656
657 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
658 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
659 final Set<ContentAddition.Summary> contentAddSummary =
660 incomingContentAdd == null
661 ? Collections.emptySet()
662 : ContentAddition.summary(incomingContentAdd);
663 final Set<ContentAddition.Summary> removeSummary =
664 ContentAddition.summary(receivedContentRemove);
665 if (contentAddSummary.equals(removeSummary)) {
666 this.incomingContentAdd = null;
667 updateEndUserState();
668 } else {
669 webRTCWrapper.close();
670 sendSessionTerminate(
671 Reason.FAILED_APPLICATION,
672 String.format(
673 "%s only supports %s as a means to retract a not yet accepted %s",
674 BuildConfig.APP_NAME,
675 Jingle.Action.CONTENT_REMOVE,
676 Jingle.Action.CONTENT_ADD));
677 }
678 }
679
680 public synchronized void retractContentAdd() {
681 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
682 if (outgoingContentAdd == null) {
683 throw new IllegalStateException("Not outgoing content add");
684 }
685 try {
686 webRTCWrapper.removeTrack(Media.VIDEO);
687 final RtpContentMap localContentMap = customRollback();
688 modifyLocalContentMap(localContentMap);
689 } catch (final Exception e) {
690 final Throwable cause = Throwables.getRootCause(e);
691 Log.d(
692 Config.LOGTAG,
693 id.getAccount().getJid().asBareJid()
694 + ": unable to rollback local description after trying to retract"
695 + " content-add",
696 cause);
697 webRTCWrapper.close();
698 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
699 return;
700 }
701 this.outgoingContentAdd = null;
702 final Iq retract =
703 outgoingContentAdd
704 .toStub()
705 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
706 this.send(retract);
707 Log.d(
708 Config.LOGTAG,
709 id.getAccount().getJid()
710 + ": retract content-add "
711 + ContentAddition.summary(outgoingContentAdd));
712 }
713
714 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
715 final SessionDescription sdp = setLocalSessionDescription();
716 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
717 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
718 this.webRTCWrapper
719 .setRemoteDescription(
720 new org.webrtc.SessionDescription(
721 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
722 .get();
723 return localRtpContentMap;
724 }
725
726 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
727 final RtpContentMap currentRemote = getRemoteContentMap();
728 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
729 if (diff.isEmpty()) {
730 throw new IllegalStateException(
731 "Unexpected rollback condition. No difference between local and remote");
732 }
733 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
734 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
735 final RtpContentMap nextRemote =
736 currentRemote.addContent(
737 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
738 return SessionDescription.of(nextRemote, isResponder());
739 }
740 throw new IllegalStateException(
741 "Unexpected rollback condition. Senders were not uniformly none");
742 }
743
744 public synchronized void acceptContentAdd(
745 @NonNull final Set<ContentAddition.Summary> contentAddition) {
746 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
747 if (incomingContentAdd == null) {
748 throw new IllegalStateException("No incoming content add");
749 }
750
751 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
752 this.incomingContentAdd = null;
753 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
754 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
755 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
756 Log.d(
757 Config.LOGTAG,
758 "content addition is receive only. we want to upgrade to 'both'");
759 final RtpContentMap modifiedSenders =
760 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
761 final Iq proposedContentModification =
762 modifiedSenders
763 .toStub()
764 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
765 proposedContentModification.setTo(id.with);
766 xmppConnectionService.sendIqPacket(
767 id.account,
768 proposedContentModification,
769 (response) -> {
770 if (response.getType() == Iq.Type.RESULT) {
771 Log.d(
772 Config.LOGTAG,
773 id.account.getJid().asBareJid()
774 + ": remote has accepted our upgrade to"
775 + " senders=both");
776 acceptContentAdd(
777 ContentAddition.summary(modifiedSenders), modifiedSenders);
778 } else {
779 Log.d(
780 Config.LOGTAG,
781 id.account.getJid().asBareJid()
782 + ": remote has rejected our upgrade to"
783 + " senders=both");
784 acceptContentAdd(contentAddition, incomingContentAdd);
785 }
786 });
787 } else {
788 acceptContentAdd(contentAddition, incomingContentAdd);
789 }
790 } else {
791 throw new IllegalStateException(
792 "Accepted content add does not match pending content-add");
793 }
794 }
795
796 private void acceptContentAdd(
797 @NonNull final Set<ContentAddition.Summary> contentAddition,
798 final RtpContentMap incomingContentAdd) {
799 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
800 final RtpContentMap modifiedContentMap =
801 getRemoteContentMap().addContent(incomingContentAdd, setup);
802 this.setRemoteContentMap(modifiedContentMap);
803
804 final SessionDescription offer;
805 try {
806 offer = SessionDescription.of(modifiedContentMap, isResponder());
807 } catch (final IllegalArgumentException | NullPointerException e) {
808 Log.d(
809 Config.LOGTAG,
810 id.getAccount().getJid().asBareJid()
811 + ": unable convert offer from content-add to SDP",
812 e);
813 webRTCWrapper.close();
814 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
815 return;
816 }
817 this.incomingContentAdd = null;
818 acceptContentAdd(contentAddition, offer);
819 }
820
821 private void acceptContentAdd(
822 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
823 final org.webrtc.SessionDescription sdp =
824 new org.webrtc.SessionDescription(
825 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
826 try {
827 this.webRTCWrapper.setRemoteDescription(sdp).get();
828
829 // TODO add tracks for 'media' where contentAddition.senders matches
830
831 // TODO if senders.sending(isInitiator())
832
833 this.webRTCWrapper.addTrack(Media.VIDEO);
834
835 // TODO add additional transceivers for recv only cases
836
837 final SessionDescription answer = setLocalSessionDescription();
838 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
839
840 final RtpContentMap contentAcceptMap =
841 rtpContentMap.toContentModification(
842 Collections2.transform(contentAddition, ca -> ca.name));
843
844 Log.d(
845 Config.LOGTAG,
846 id.getAccount().getJid().asBareJid()
847 + ": sending content-accept "
848 + ContentAddition.summary(contentAcceptMap));
849
850 addIceCandidatesFromBlackLog();
851
852 modifyLocalContentMap(rtpContentMap);
853 final ListenableFuture<RtpContentMap> future =
854 prepareOutgoingContentMap(contentAcceptMap);
855 Futures.addCallback(
856 future,
857 new FutureCallback<>() {
858 @Override
859 public void onSuccess(final RtpContentMap rtpContentMap) {
860 sendContentAccept(rtpContentMap);
861 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
862 }
863
864 @Override
865 public void onFailure(@NonNull final Throwable throwable) {
866 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
867 }
868 },
869 MoreExecutors.directExecutor());
870 } catch (final Exception e) {
871 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
872 webRTCWrapper.close();
873 sendSessionTerminate(Reason.FAILED_APPLICATION);
874 }
875 }
876
877 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
878 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
879 send(iq);
880 }
881
882 public synchronized void rejectContentAdd() {
883 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
884 if (incomingContentAdd == null) {
885 throw new IllegalStateException("No incoming content add");
886 }
887 this.incomingContentAdd = null;
888 updateEndUserState();
889 rejectContentAdd(incomingContentAdd);
890 }
891
892 private void rejectContentAdd(final RtpContentMap contentMap) {
893 final Iq iq =
894 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
895 Log.d(
896 Config.LOGTAG,
897 id.getAccount().getJid().asBareJid()
898 + ": rejecting content "
899 + ContentAddition.summary(contentMap));
900 send(iq);
901 }
902
903 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
904 final RtpContentMap existing = getRemoteContentMap();
905 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
906 final IceUdpTransportInfo.Credentials newCredentials;
907 try {
908 existingCredentials = existing.getCredentials();
909 newCredentials = rtpContentMap.getDistinctCredentials();
910 } catch (final IllegalStateException e) {
911 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
912 return false;
913 }
914 if (existingCredentials.contains(newCredentials)) {
915 return false;
916 }
917 // TODO an alternative approach is to check if we already got an iq result to our
918 // ICE-restart
919 // and if that's the case we are seeing an answer.
920 // This might be more spec compliant but also more error prone potentially
921 final boolean isSignalStateStable =
922 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
923 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
924 final boolean isOffer = rtpContentMap.emptyCandidates();
925 final RtpContentMap restartContentMap;
926 try {
927 if (isOffer) {
928 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
929 restartContentMap =
930 existing.modifiedCredentials(
931 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
932 } else {
933 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
934 Log.d(
935 Config.LOGTAG,
936 "received confirmation of ICE restart"
937 + newCredentials
938 + " peer_setup="
939 + setup);
940 // DTLS setup attribute needs to be rewritten to reflect current peer state
941 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
942 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
943 }
944 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
945 return isOffer;
946 } else {
947 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
948 respondWithTieBreak(jinglePacket);
949 return true;
950 }
951 } catch (final Exception exception) {
952 respondOk(jinglePacket);
953 final Throwable rootCause = Throwables.getRootCause(exception);
954 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
955 // If this happens a termination is already in progress
956 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
957 return true;
958 }
959 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
960 webRTCWrapper.close();
961 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
962 return true;
963 }
964 }
965
966 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
967 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
968 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
969 throw new IllegalStateException("Invalid peer setup");
970 }
971 return peerSetup;
972 }
973
974 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
975 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
976 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
977 }
978 this.peerDtlsSetup = setup;
979 }
980
981 private boolean applyIceRestart(
982 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
983 throws ExecutionException, InterruptedException {
984 final SessionDescription sessionDescription =
985 SessionDescription.of(restartContentMap, isResponder());
986 final org.webrtc.SessionDescription.Type type =
987 isOffer
988 ? org.webrtc.SessionDescription.Type.OFFER
989 : org.webrtc.SessionDescription.Type.ANSWER;
990 org.webrtc.SessionDescription sdp =
991 new org.webrtc.SessionDescription(type, sessionDescription.toString());
992 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
993 if (isInitiator()) {
994 // We ignore the offer and respond with tie-break. This will clause the responder
995 // not to apply the content map
996 return false;
997 }
998 }
999 webRTCWrapper.setRemoteDescription(sdp).get();
1000 setRemoteContentMap(restartContentMap);
1001 if (isOffer) {
1002 final SessionDescription localSessionDescription = setLocalSessionDescription();
1003 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1004 // We need to respond OK before sending any candidates
1005 respondOk(jinglePacket);
1006 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1007 } else {
1008 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1009 }
1010 return true;
1011 }
1012
1013 private void processCandidates(
1014 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1015 contents) {
1016 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1017 content : contents) {
1018 processCandidate(content);
1019 }
1020 }
1021
1022 private void processCandidate(
1023 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1024 content) {
1025 final RtpContentMap rtpContentMap = getRemoteContentMap();
1026 final List<String> indices = toIdentificationTags(rtpContentMap);
1027 final String sdpMid = content.getKey(); // aka content name
1028 final IceUdpTransportInfo transport = content.getValue().transport;
1029 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1030
1031 // TODO check that credentials remained the same
1032
1033 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1034 final String sdp;
1035 try {
1036 sdp = candidate.toSdpAttribute(credentials.ufrag);
1037 } catch (final IllegalArgumentException e) {
1038 Log.d(
1039 Config.LOGTAG,
1040 id.account.getJid().asBareJid()
1041 + ": ignoring invalid ICE candidate "
1042 + e.getMessage());
1043 continue;
1044 }
1045 final int mLineIndex = indices.indexOf(sdpMid);
1046 if (mLineIndex < 0) {
1047 Log.w(
1048 Config.LOGTAG,
1049 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1050 }
1051 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1052 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1053 this.webRTCWrapper.addIceCandidate(iceCandidate);
1054 }
1055 }
1056
1057 private RtpContentMap getRemoteContentMap() {
1058 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1059 }
1060
1061 private RtpContentMap getLocalContentMap() {
1062 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1063 }
1064
1065 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1066 final Group originalGroup = rtpContentMap.group;
1067 final List<String> identificationTags =
1068 originalGroup == null
1069 ? rtpContentMap.getNames()
1070 : originalGroup.getIdentificationTags();
1071 if (identificationTags.size() == 0) {
1072 Log.w(
1073 Config.LOGTAG,
1074 id.account.getJid().asBareJid()
1075 + ": no identification tags found in initial offer. we won't be able to"
1076 + " calculate mLineIndices");
1077 }
1078 return identificationTags;
1079 }
1080
1081 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1082 final Jingle jinglePacket, final boolean expectVerification) {
1083 try {
1084 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1085 } catch (final Exception e) {
1086 return Futures.immediateFailedFuture(e);
1087 }
1088 }
1089
1090 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1091 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1092 Log.d(
1093 Config.LOGTAG,
1094 "receiveRtpContentMap("
1095 + receivedContentMap.getClass().getSimpleName()
1096 + ",expectVerification="
1097 + expectVerification
1098 + ")");
1099 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1100 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1101 id.account
1102 .getAxolotlService()
1103 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1104 return Futures.transform(
1105 future,
1106 omemoVerifiedPayload -> {
1107 // TODO test if an exception here triggers a correct abort
1108 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1109 Log.d(
1110 Config.LOGTAG,
1111 id.account.getJid().asBareJid()
1112 + ": received verifiable DTLS fingerprint via "
1113 + omemoVerification);
1114 return omemoVerifiedPayload.getPayload();
1115 },
1116 MoreExecutors.directExecutor());
1117 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1118 return Futures.immediateFailedFuture(
1119 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1120 } else {
1121 return Futures.immediateFuture(receivedContentMap);
1122 }
1123 }
1124
1125 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1126 if (isInitiator()) {
1127 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1128 return;
1129 }
1130 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1131 Futures.addCallback(
1132 future,
1133 new FutureCallback<>() {
1134 @Override
1135 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1136 receiveSessionInitiate(jinglePacket, rtpContentMap);
1137 }
1138
1139 @Override
1140 public void onFailure(@NonNull final Throwable throwable) {
1141 respondOk(jinglePacket);
1142 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1143 }
1144 },
1145 MoreExecutors.directExecutor());
1146 }
1147
1148 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1149 try {
1150 contentMap.requireContentDescriptions();
1151 contentMap.requireDTLSFingerprint(true);
1152 } catch (final RuntimeException e) {
1153 Log.d(
1154 Config.LOGTAG,
1155 id.account.getJid().asBareJid() + ": improperly formatted contents",
1156 Throwables.getRootCause(e));
1157 respondOk(jinglePacket);
1158 sendSessionTerminate(Reason.of(e), e.getMessage());
1159 return;
1160 }
1161 Log.d(
1162 Config.LOGTAG,
1163 "processing session-init with " + contentMap.contents.size() + " contents");
1164 final State target;
1165 if (this.state == State.PROCEED) {
1166 Preconditions.checkState(
1167 proposedMedia != null && proposedMedia.size() > 0,
1168 "proposed media must be set when processing pre-approved session-initiate");
1169 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1170 sendSessionTerminate(
1171 Reason.SECURITY_ERROR,
1172 String.format(
1173 "Your session proposal (Jingle Message Initiation) included media"
1174 + " %s but your session-initiate was %s",
1175 this.proposedMedia, contentMap.getMedia()));
1176 return;
1177 }
1178 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1179 } else {
1180 target = State.SESSION_INITIALIZED;
1181 setProposedMedia(contentMap.getMedia());
1182 }
1183 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1184 respondOk(jinglePacket);
1185 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1186 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1187 Log.d(
1188 Config.LOGTAG,
1189 id.account.getJid().asBareJid()
1190 + ": automatically accepting session-initiate");
1191 sendSessionAccept();
1192 } else {
1193 Log.d(
1194 Config.LOGTAG,
1195 id.account.getJid().asBareJid()
1196 + ": received not pre-approved session-initiate. start ringing");
1197 startRinging();
1198 }
1199 } else {
1200 Log.d(
1201 Config.LOGTAG,
1202 String.format(
1203 "%s: received session-initiate while in state %s",
1204 id.account.getJid().asBareJid(), state));
1205 terminateWithOutOfOrder(jinglePacket);
1206 }
1207 }
1208
1209 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1210 if (isResponder()) {
1211 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1212 return;
1213 }
1214 final ListenableFuture<RtpContentMap> future =
1215 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1216 Futures.addCallback(
1217 future,
1218 new FutureCallback<>() {
1219 @Override
1220 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1221 receiveSessionAccept(jinglePacket, rtpContentMap);
1222 }
1223
1224 @Override
1225 public void onFailure(@NonNull final Throwable throwable) {
1226 respondOk(jinglePacket);
1227 Log.d(
1228 Config.LOGTAG,
1229 id.account.getJid().asBareJid()
1230 + ": improperly formatted contents in session-accept",
1231 throwable);
1232 webRTCWrapper.close();
1233 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1234 }
1235 },
1236 MoreExecutors.directExecutor());
1237 }
1238
1239 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1240 try {
1241 contentMap.requireContentDescriptions();
1242 contentMap.requireDTLSFingerprint();
1243 } catch (final RuntimeException e) {
1244 respondOk(jinglePacket);
1245 Log.d(
1246 Config.LOGTAG,
1247 id.account.getJid().asBareJid()
1248 + ": improperly formatted contents in session-accept",
1249 e);
1250 webRTCWrapper.close();
1251 sendSessionTerminate(Reason.of(e), e.getMessage());
1252 return;
1253 }
1254 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1255 if (!initiatorMedia.equals(contentMap.getMedia())) {
1256 sendSessionTerminate(
1257 Reason.SECURITY_ERROR,
1258 String.format(
1259 "Your session-included included media %s but our session-initiate was"
1260 + " %s",
1261 this.proposedMedia, contentMap.getMedia()));
1262 return;
1263 }
1264 Log.d(
1265 Config.LOGTAG,
1266 "processing session-accept with " + contentMap.contents.size() + " contents");
1267 if (transition(State.SESSION_ACCEPTED)) {
1268 respondOk(jinglePacket);
1269 receiveSessionAccept(contentMap);
1270 } else {
1271 Log.d(
1272 Config.LOGTAG,
1273 String.format(
1274 "%s: received session-accept while in state %s",
1275 id.account.getJid().asBareJid(), state));
1276 respondOk(jinglePacket);
1277 }
1278 }
1279
1280 private void receiveSessionAccept(final RtpContentMap contentMap) {
1281 this.responderRtpContentMap = contentMap;
1282 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1283 final SessionDescription sessionDescription;
1284 try {
1285 sessionDescription = SessionDescription.of(contentMap, false);
1286 } catch (final IllegalArgumentException | NullPointerException e) {
1287 Log.d(
1288 Config.LOGTAG,
1289 id.account.getJid().asBareJid()
1290 + ": unable convert offer from session-accept to SDP",
1291 e);
1292 webRTCWrapper.close();
1293 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1294 return;
1295 }
1296 final org.webrtc.SessionDescription answer =
1297 new org.webrtc.SessionDescription(
1298 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1299 try {
1300 this.webRTCWrapper.setRemoteDescription(answer).get();
1301 } catch (final Exception e) {
1302 Log.d(
1303 Config.LOGTAG,
1304 id.account.getJid().asBareJid()
1305 + ": unable to set remote description after receiving session-accept",
1306 Throwables.getRootCause(e));
1307 webRTCWrapper.close();
1308 sendSessionTerminate(
1309 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1310 return;
1311 }
1312 processCandidates(contentMap.contents.entrySet());
1313 }
1314
1315 private void sendSessionAccept() {
1316 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1317 if (rtpContentMap == null) {
1318 throw new IllegalStateException("initiator RTP Content Map has not been set");
1319 }
1320 final SessionDescription offer;
1321 try {
1322 offer = SessionDescription.of(rtpContentMap, true);
1323 } catch (final IllegalArgumentException | NullPointerException e) {
1324 Log.d(
1325 Config.LOGTAG,
1326 id.account.getJid().asBareJid()
1327 + ": unable convert offer from session-initiate to SDP",
1328 e);
1329 webRTCWrapper.close();
1330 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1331 return;
1332 }
1333 sendSessionAccept(rtpContentMap.getMedia(), offer);
1334 }
1335
1336 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1337 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1338 }
1339
1340 private synchronized void sendSessionAccept(
1341 final Set<Media> media,
1342 final SessionDescription offer,
1343 final Collection<PeerConnection.IceServer> iceServers) {
1344 if (isTerminated()) {
1345 Log.w(
1346 Config.LOGTAG,
1347 id.account.getJid().asBareJid()
1348 + ": ICE servers got discovered when session was already terminated."
1349 + " nothing to do.");
1350 return;
1351 }
1352 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1353 try {
1354 setupWebRTC(media, iceServers, !includeCandidates);
1355 } catch (final WebRTCWrapper.InitializationException e) {
1356 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1357 webRTCWrapper.close();
1358 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1359 return;
1360 }
1361 final org.webrtc.SessionDescription sdp =
1362 new org.webrtc.SessionDescription(
1363 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1364 try {
1365 this.webRTCWrapper.setRemoteDescription(sdp).get();
1366 addIceCandidatesFromBlackLog();
1367 org.webrtc.SessionDescription webRTCSessionDescription =
1368 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1369 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1370 } catch (final Exception e) {
1371 failureToAcceptSession(e);
1372 }
1373 }
1374
1375 private void failureToAcceptSession(final Throwable throwable) {
1376 if (isTerminated()) {
1377 return;
1378 }
1379 final Throwable rootCause = Throwables.getRootCause(throwable);
1380 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1381 webRTCWrapper.close();
1382 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1383 }
1384
1385 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1386 if (isTerminated()) {
1387 return;
1388 }
1389 final Throwable rootCause = Throwables.getRootCause(throwable);
1390 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1391 webRTCWrapper.close();
1392 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1393 }
1394
1395 private void addIceCandidatesFromBlackLog() {
1396 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1397 while ((foo = this.pendingIceCandidates.poll()) != null) {
1398 processCandidate(foo);
1399 Log.d(
1400 Config.LOGTAG,
1401 id.account.getJid().asBareJid() + ": added candidate from back log");
1402 }
1403 }
1404
1405 private void prepareSessionAccept(
1406 final org.webrtc.SessionDescription webRTCSessionDescription,
1407 final boolean includeCandidates) {
1408 final SessionDescription sessionDescription =
1409 SessionDescription.parse(webRTCSessionDescription.description);
1410 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1411 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1412 if (includeCandidates) {
1413 candidates = parseCandidates(sessionDescription);
1414 } else {
1415 candidates = ImmutableMultimap.of();
1416 }
1417 this.responderRtpContentMap = respondingRtpContentMap;
1418 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1419 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1420 prepareOutgoingContentMap(respondingRtpContentMap);
1421 Futures.addCallback(
1422 outgoingContentMapFuture,
1423 new FutureCallback<>() {
1424 @Override
1425 public void onSuccess(final RtpContentMap outgoingContentMap) {
1426 if (includeCandidates) {
1427 Log.d(
1428 Config.LOGTAG,
1429 "including "
1430 + candidates.size()
1431 + " candidates in session accept");
1432 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1433 } else {
1434 sendSessionAccept(outgoingContentMap);
1435 }
1436 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1437 }
1438
1439 @Override
1440 public void onFailure(@NonNull Throwable throwable) {
1441 failureToAcceptSession(throwable);
1442 }
1443 },
1444 MoreExecutors.directExecutor());
1445 }
1446
1447 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1448 if (isTerminated()) {
1449 Log.w(
1450 Config.LOGTAG,
1451 id.account.getJid().asBareJid()
1452 + ": preparing session accept was too slow. already terminated. nothing"
1453 + " to do.");
1454 return;
1455 }
1456 transitionOrThrow(State.SESSION_ACCEPTED);
1457 final Iq sessionAccept =
1458 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1459 send(sessionAccept);
1460 }
1461
1462 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1463 final RtpContentMap rtpContentMap) {
1464 if (this.omemoVerification.hasDeviceId()) {
1465 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1466 verifiedPayloadFuture =
1467 id.account
1468 .getAxolotlService()
1469 .encrypt(
1470 rtpContentMap,
1471 id.with,
1472 omemoVerification.getDeviceId());
1473 return Futures.transform(
1474 verifiedPayloadFuture,
1475 verifiedPayload -> {
1476 omemoVerification.setOrEnsureEqual(verifiedPayload);
1477 return verifiedPayload.getPayload();
1478 },
1479 MoreExecutors.directExecutor());
1480 } else {
1481 return Futures.immediateFuture(rtpContentMap);
1482 }
1483 }
1484
1485 synchronized void deliveryMessage(
1486 final Jid from,
1487 final Element message,
1488 final String serverMessageId,
1489 final long timestamp) {
1490 Log.d(
1491 Config.LOGTAG,
1492 id.account.getJid().asBareJid()
1493 + ": delivered message to JingleRtpConnection "
1494 + message);
1495 switch (message.getName()) {
1496 case "propose" ->
1497 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1498 case "proceed" ->
1499 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1500 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1501 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1502 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1503 }
1504 }
1505
1506 void deliverFailedProceed(final String message) {
1507 Log.d(
1508 Config.LOGTAG,
1509 id.account.getJid().asBareJid()
1510 + ": receive message error for proceed message ("
1511 + Strings.nullToEmpty(message)
1512 + ")");
1513 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1514 webRTCWrapper.close();
1515 Log.d(
1516 Config.LOGTAG,
1517 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1518 this.finish();
1519 }
1520 }
1521
1522 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1523 final boolean originatedFromMyself =
1524 from.asBareJid().equals(id.account.getJid().asBareJid());
1525 if (originatedFromMyself) {
1526 if (transition(State.ACCEPTED)) {
1527 acceptedOnOtherDevice(serverMsgId, timestamp);
1528 } else {
1529 Log.d(
1530 Config.LOGTAG,
1531 id.account.getJid().asBareJid()
1532 + ": unable to transition to accept because already in state="
1533 + this.state);
1534 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1535 }
1536 } else {
1537 Log.d(
1538 Config.LOGTAG,
1539 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1540 }
1541 }
1542
1543 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1544 if (serverMsgId != null) {
1545 this.message.setServerMsgId(serverMsgId);
1546 }
1547 this.message.setTime(timestamp);
1548 this.message.setCarbon(true); // indicate that call was accepted on other device
1549 this.writeLogMessageSuccess(0);
1550 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1551 this.finish();
1552 }
1553
1554 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1555 final boolean originatedFromMyself =
1556 from.asBareJid().equals(id.account.getJid().asBareJid());
1557 // reject from another one of my clients
1558 if (originatedFromMyself) {
1559 receiveRejectFromMyself(serverMsgId, timestamp);
1560 } else if (isInitiator()) {
1561 if (from.equals(id.with)) {
1562 receiveRejectFromResponder();
1563 } else {
1564 Log.d(
1565 Config.LOGTAG,
1566 id.account.getJid()
1567 + ": ignoring reject from "
1568 + from
1569 + " for session with "
1570 + id.with);
1571 }
1572 } else {
1573 Log.d(
1574 Config.LOGTAG,
1575 id.account.getJid()
1576 + ": ignoring reject from "
1577 + from
1578 + " for session with "
1579 + id.with);
1580 }
1581 }
1582
1583 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1584 if (transition(State.REJECTED)) {
1585 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1586 this.finish();
1587 if (serverMsgId != null) {
1588 this.message.setServerMsgId(serverMsgId);
1589 }
1590 this.message.setTime(timestamp);
1591 this.message.setCarbon(true); // indicate that call was rejected on other device
1592 writeLogMessageMissed();
1593 } else {
1594 Log.d(
1595 Config.LOGTAG,
1596 "not able to transition into REJECTED because already in " + this.state);
1597 }
1598 }
1599
1600 private void receiveRejectFromResponder() {
1601 if (isInState(State.PROCEED)) {
1602 Log.d(
1603 Config.LOGTAG,
1604 id.account.getJid()
1605 + ": received reject while still in proceed. callee reconsidered");
1606 closeTransitionLogFinish(State.REJECTED_RACED);
1607 return;
1608 }
1609 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1610 Log.d(
1611 Config.LOGTAG,
1612 id.account.getJid()
1613 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee"
1614 + " reconsidered before receiving session-init");
1615 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1616 return;
1617 }
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": ignoring reject from responder because already in state "
1622 + this.state);
1623 }
1624
1625 private void receivePropose(
1626 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1627 final boolean originatedFromMyself =
1628 from.asBareJid().equals(id.account.getJid().asBareJid());
1629 if (originatedFromMyself) {
1630 Log.d(
1631 Config.LOGTAG,
1632 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1633 } else if (transition(
1634 State.PROPOSED,
1635 () -> {
1636 final Collection<RtpDescription> descriptions =
1637 Collections2.transform(
1638 Collections2.filter(
1639 propose.getDescriptions(),
1640 d -> d instanceof RtpDescription),
1641 input -> (RtpDescription) input);
1642 final Collection<Media> media =
1643 Collections2.transform(descriptions, RtpDescription::getMedia);
1644 Preconditions.checkState(
1645 !media.contains(Media.UNKNOWN),
1646 "RTP descriptions contain unknown media");
1647 Log.d(
1648 Config.LOGTAG,
1649 id.account.getJid().asBareJid()
1650 + ": received session proposal from "
1651 + from
1652 + " for "
1653 + media);
1654 this.setProposedMedia(Sets.newHashSet(media));
1655 })) {
1656 if (serverMsgId != null) {
1657 this.message.setServerMsgId(serverMsgId);
1658 }
1659 this.message.setTime(timestamp);
1660 startRinging();
1661 // in environments where we always use discovery timeouts we always want to respond with
1662 // 'ringing'
1663 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1664 || id.getContact().showInContactList()) {
1665 sendJingleMessage("ringing");
1666 }
1667 } else {
1668 Log.d(
1669 Config.LOGTAG,
1670 id.account.getJid()
1671 + ": ignoring session proposal because already in "
1672 + state);
1673 }
1674 }
1675
1676 private void startRinging() {
1677 this.callIntegration.setRinging();
1678 Log.d(
1679 Config.LOGTAG,
1680 id.account.getJid().asBareJid()
1681 + ": received call from "
1682 + id.with
1683 + ". start ringing");
1684 ringingTimeoutFuture =
1685 jingleConnectionManager.schedule(
1686 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1687 if (CallIntegration.selfManaged(xmppConnectionService)) {
1688 return;
1689 }
1690 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1691 }
1692
1693 private synchronized void ringingTimeout() {
1694 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1695 switch (this.state) {
1696 case PROPOSED -> {
1697 message.markUnread();
1698 rejectCallFromProposed();
1699 }
1700 case SESSION_INITIALIZED -> {
1701 message.markUnread();
1702 rejectCallFromSessionInitiate();
1703 }
1704 }
1705 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1706 }
1707
1708 private void cancelRingingTimeout() {
1709 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1710 if (future != null && !future.isCancelled()) {
1711 future.cancel(false);
1712 }
1713 }
1714
1715 private void receiveProceed(
1716 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1717 final Set<Media> media =
1718 Preconditions.checkNotNull(
1719 this.proposedMedia, "Proposed media has to be set before handling proceed");
1720 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1721 if (from.equals(id.with)) {
1722 if (isInitiator()) {
1723 if (transition(State.PROCEED)) {
1724 if (serverMsgId != null) {
1725 this.message.setServerMsgId(serverMsgId);
1726 }
1727 this.message.setTime(timestamp);
1728 final Integer remoteDeviceId = proceed.getDeviceId();
1729 if (isOmemoEnabled()) {
1730 this.omemoVerification.setDeviceId(remoteDeviceId);
1731 } else {
1732 if (remoteDeviceId != null) {
1733 Log.d(
1734 Config.LOGTAG,
1735 id.account.getJid().asBareJid()
1736 + ": remote party signaled support for OMEMO"
1737 + " verification but we have OMEMO disabled");
1738 }
1739 this.omemoVerification.setDeviceId(null);
1740 }
1741 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1742 } else {
1743 Log.d(
1744 Config.LOGTAG,
1745 String.format(
1746 "%s: ignoring proceed because already in %s",
1747 id.account.getJid().asBareJid(), this.state));
1748 }
1749 } else {
1750 Log.d(
1751 Config.LOGTAG,
1752 String.format(
1753 "%s: ignoring proceed because we were not initializing",
1754 id.account.getJid().asBareJid()));
1755 }
1756 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1757 if (transition(State.ACCEPTED)) {
1758 Log.d(
1759 Config.LOGTAG,
1760 id.account.getJid().asBareJid()
1761 + ": moved session with "
1762 + id.with
1763 + " into state accepted after received carbon copied proceed");
1764 acceptedOnOtherDevice(serverMsgId, timestamp);
1765 }
1766 } else {
1767 Log.d(
1768 Config.LOGTAG,
1769 String.format(
1770 "%s: ignoring proceed from %s. was expected from %s",
1771 id.account.getJid().asBareJid(), from, id.with));
1772 }
1773 }
1774
1775 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1776 if (from.equals(id.with)) {
1777 final State target =
1778 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1779 if (transition(target)) {
1780 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1781 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1782 Log.d(
1783 Config.LOGTAG,
1784 id.account.getJid().asBareJid()
1785 + ": session with "
1786 + id.with
1787 + " has been retracted (serverMsgId="
1788 + serverMsgId
1789 + ")");
1790 if (serverMsgId != null) {
1791 this.message.setServerMsgId(serverMsgId);
1792 }
1793 this.message.setTime(timestamp);
1794 if (target == State.RETRACTED) {
1795 this.message.markUnread();
1796 }
1797 writeLogMessageMissed();
1798 finish();
1799 } else {
1800 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1801 }
1802 } else {
1803 // TODO parse retract from self
1804 Log.d(
1805 Config.LOGTAG,
1806 id.account.getJid().asBareJid()
1807 + ": received retract from "
1808 + from
1809 + ". expected retract from"
1810 + id.with
1811 + ". ignoring");
1812 }
1813 }
1814
1815 public void sendSessionInitiate() {
1816 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1817 }
1818
1819 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1820 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1821 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1822 }
1823
1824 private synchronized void sendSessionInitiate(
1825 final Set<Media> media,
1826 final State targetState,
1827 final Collection<PeerConnection.IceServer> iceServers) {
1828 if (isTerminated()) {
1829 Log.w(
1830 Config.LOGTAG,
1831 id.account.getJid().asBareJid()
1832 + ": ICE servers got discovered when session was already terminated."
1833 + " nothing to do.");
1834 return;
1835 }
1836 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1837 try {
1838 setupWebRTC(media, iceServers, !includeCandidates);
1839 } catch (final WebRTCWrapper.InitializationException e) {
1840 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1841 webRTCWrapper.close();
1842 sendRetract(Reason.ofThrowable(e));
1843 return;
1844 }
1845 try {
1846 org.webrtc.SessionDescription webRTCSessionDescription =
1847 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1848 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1849 } catch (final Exception e) {
1850 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1851 // exceptions
1852 failureToInitiateSession(e, targetState);
1853 }
1854 }
1855
1856 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1857 if (isTerminated()) {
1858 return;
1859 }
1860 Log.d(
1861 Config.LOGTAG,
1862 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1863 Throwables.getRootCause(throwable));
1864 webRTCWrapper.close();
1865 final Reason reason = Reason.ofThrowable(throwable);
1866 if (isInState(targetState)) {
1867 sendSessionTerminate(reason, throwable.getMessage());
1868 } else {
1869 sendRetract(reason);
1870 }
1871 }
1872
1873 private void sendRetract(final Reason reason) {
1874 // TODO embed reason into retract
1875 sendJingleMessage("retract", id.with.asBareJid());
1876 transitionOrThrow(reasonToState(reason));
1877 this.finish();
1878 }
1879
1880 private void prepareSessionInitiate(
1881 final org.webrtc.SessionDescription webRTCSessionDescription,
1882 final boolean includeCandidates,
1883 final State targetState) {
1884 final SessionDescription sessionDescription =
1885 SessionDescription.parse(webRTCSessionDescription.description);
1886 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1887 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1888 if (includeCandidates) {
1889 candidates = parseCandidates(sessionDescription);
1890 } else {
1891 candidates = ImmutableMultimap.of();
1892 }
1893 this.initiatorRtpContentMap = rtpContentMap;
1894 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1895 encryptSessionInitiate(rtpContentMap);
1896 Futures.addCallback(
1897 outgoingContentMapFuture,
1898 new FutureCallback<>() {
1899 @Override
1900 public void onSuccess(final RtpContentMap outgoingContentMap) {
1901 if (includeCandidates) {
1902 Log.d(
1903 Config.LOGTAG,
1904 "including "
1905 + candidates.size()
1906 + " candidates in session initiate");
1907 sendSessionInitiate(
1908 outgoingContentMap.withCandidates(candidates), targetState);
1909 } else {
1910 sendSessionInitiate(outgoingContentMap, targetState);
1911 }
1912 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1913 }
1914
1915 @Override
1916 public void onFailure(@NonNull final Throwable throwable) {
1917 failureToInitiateSession(throwable, targetState);
1918 }
1919 },
1920 MoreExecutors.directExecutor());
1921 }
1922
1923 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1924 if (isTerminated()) {
1925 Log.w(
1926 Config.LOGTAG,
1927 id.account.getJid().asBareJid()
1928 + ": preparing session was too slow. already terminated. nothing to"
1929 + " do.");
1930 return;
1931 }
1932 this.transitionOrThrow(targetState);
1933 final Iq sessionInitiate =
1934 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1935 send(sessionInitiate);
1936 }
1937
1938 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1939 final RtpContentMap rtpContentMap) {
1940 if (this.omemoVerification.hasDeviceId()) {
1941 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1942 verifiedPayloadFuture =
1943 id.account
1944 .getAxolotlService()
1945 .encrypt(
1946 rtpContentMap,
1947 id.with,
1948 omemoVerification.getDeviceId());
1949 final ListenableFuture<RtpContentMap> future =
1950 Futures.transform(
1951 verifiedPayloadFuture,
1952 verifiedPayload -> {
1953 omemoVerification.setSessionFingerprint(
1954 verifiedPayload.getFingerprint());
1955 return verifiedPayload.getPayload();
1956 },
1957 MoreExecutors.directExecutor());
1958 if (Config.REQUIRE_RTP_VERIFICATION) {
1959 return future;
1960 }
1961 return Futures.catching(
1962 future,
1963 CryptoFailedException.class,
1964 e -> {
1965 Log.w(
1966 Config.LOGTAG,
1967 id.account.getJid().asBareJid()
1968 + ": unable to use OMEMO DTLS verification on outgoing"
1969 + " session initiate. falling back",
1970 e);
1971 return rtpContentMap;
1972 },
1973 MoreExecutors.directExecutor());
1974 } else {
1975 return Futures.immediateFuture(rtpContentMap);
1976 }
1977 }
1978
1979 protected void sendSessionTerminate(final Reason reason) {
1980 sendSessionTerminate(reason, null);
1981 }
1982
1983 protected void sendSessionTerminate(final Reason reason, final String text) {
1984 sendSessionTerminate(reason, text, this::writeLogMessage);
1985 sendJingleMessageFinish(reason);
1986 }
1987
1988 private void sendTransportInfo(
1989 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1990 final RtpContentMap transportInfo;
1991 try {
1992 final RtpContentMap rtpContentMap =
1993 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1994 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1995 } catch (final Exception e) {
1996 Log.d(
1997 Config.LOGTAG,
1998 id.account.getJid().asBareJid()
1999 + ": unable to prepare transport-info from candidate for content="
2000 + contentName);
2001 return;
2002 }
2003 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2004 send(iq);
2005 }
2006
2007 public RtpEndUserState getEndUserState() {
2008 switch (this.state) {
2009 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2010 if (isInitiator()) {
2011 return RtpEndUserState.RINGING;
2012 } else {
2013 return RtpEndUserState.INCOMING_CALL;
2014 }
2015 }
2016 case PROCEED -> {
2017 if (isInitiator()) {
2018 return RtpEndUserState.RINGING;
2019 } else {
2020 return RtpEndUserState.ACCEPTING_CALL;
2021 }
2022 }
2023 case SESSION_INITIALIZED_PRE_APPROVED -> {
2024 if (isInitiator()) {
2025 return RtpEndUserState.RINGING;
2026 } else {
2027 return RtpEndUserState.CONNECTING;
2028 }
2029 }
2030 case SESSION_ACCEPTED -> {
2031 final ContentAddition ca = getPendingContentAddition();
2032 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2033 return RtpEndUserState.INCOMING_CONTENT_ADD;
2034 }
2035 return getPeerConnectionStateAsEndUserState();
2036 }
2037 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.DECLINED_OR_BUSY;
2040 } else {
2041 return RtpEndUserState.ENDED;
2042 }
2043 }
2044 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2045 return RtpEndUserState.ENDED;
2046 }
2047 case RETRACTED_RACED -> {
2048 if (isInitiator()) {
2049 return RtpEndUserState.ENDED;
2050 } else {
2051 return RtpEndUserState.RETRACTED;
2052 }
2053 }
2054 case TERMINATED_CONNECTIVITY_ERROR -> {
2055 return zeroDuration()
2056 ? RtpEndUserState.CONNECTIVITY_ERROR
2057 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2058 }
2059 case TERMINATED_APPLICATION_FAILURE -> {
2060 return RtpEndUserState.APPLICATION_ERROR;
2061 }
2062 case TERMINATED_SECURITY_ERROR -> {
2063 return RtpEndUserState.SECURITY_ERROR;
2064 }
2065 }
2066 throw new IllegalStateException(
2067 String.format("%s has no equivalent EndUserState", this.state));
2068 }
2069
2070 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2071 final PeerConnection.PeerConnectionState state;
2072 try {
2073 state = webRTCWrapper.getState();
2074 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2075 // We usually close the WebRTCWrapper *before* transitioning so we might still
2076 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2077 return RtpEndUserState.ENDING_CALL;
2078 }
2079 return switch (state) {
2080 case CONNECTED -> RtpEndUserState.CONNECTED;
2081 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2082 case CLOSED -> RtpEndUserState.ENDING_CALL;
2083 default ->
2084 zeroDuration()
2085 ? RtpEndUserState.CONNECTIVITY_ERROR
2086 : RtpEndUserState.RECONNECTING;
2087 };
2088 }
2089
2090 private boolean isPeerConnectionConnected() {
2091 try {
2092 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2093 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2094 return false;
2095 }
2096 }
2097
2098 private void updateCallIntegrationState() {
2099 switch (this.state) {
2100 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2101 if (isInitiator()) {
2102 this.callIntegration.setDialing();
2103 } else {
2104 this.callIntegration.setRinging();
2105 }
2106 }
2107 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2108 if (isInitiator()) {
2109 this.callIntegration.setDialing();
2110 } else {
2111 this.callIntegration.setInitialized();
2112 }
2113 }
2114 case SESSION_ACCEPTED -> {
2115 if (isPeerConnectionConnected()) {
2116 this.callIntegration.setActive();
2117 } else {
2118 this.callIntegration.setInitialized();
2119 }
2120 }
2121 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2122 if (isInitiator()) {
2123 this.callIntegration.busy();
2124 } else {
2125 this.callIntegration.rejected();
2126 }
2127 }
2128 case TERMINATED_SUCCESS -> this.callIntegration.success();
2129 case ACCEPTED -> this.callIntegration.accepted();
2130 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT ->
2131 this.callIntegration.retracted();
2132 case TERMINATED_CONNECTIVITY_ERROR,
2133 TERMINATED_APPLICATION_FAILURE,
2134 TERMINATED_SECURITY_ERROR ->
2135 this.callIntegration.error();
2136 default ->
2137 throw new IllegalStateException(String.format("%s is not handled", this.state));
2138 }
2139 }
2140
2141 public ContentAddition getPendingContentAddition() {
2142 final RtpContentMap in = this.incomingContentAdd;
2143 final RtpContentMap out = this.outgoingContentAdd;
2144 if (out != null) {
2145 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2146 } else if (in != null) {
2147 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2148 } else {
2149 return null;
2150 }
2151 }
2152
2153 public Set<Media> getMedia() {
2154 final State current = getState();
2155 if (current == State.NULL) {
2156 if (isInitiator()) {
2157 return Preconditions.checkNotNull(
2158 this.proposedMedia, "RTP connection has not been initialized properly");
2159 }
2160 throw new IllegalStateException("RTP connection has not been initialized yet");
2161 }
2162 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2163 return Preconditions.checkNotNull(
2164 this.proposedMedia, "RTP connection has not been initialized properly");
2165 }
2166 final RtpContentMap localContentMap = getLocalContentMap();
2167 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2168 if (localContentMap != null) {
2169 return localContentMap.getMedia();
2170 } else if (initiatorContentMap != null) {
2171 return initiatorContentMap.getMedia();
2172 } else if (isTerminated()) {
2173 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2174 } else {
2175 return Preconditions.checkNotNull(
2176 this.proposedMedia, "RTP connection has not been initialized properly");
2177 }
2178 }
2179
2180 public boolean isVerified() {
2181 final String fingerprint = this.omemoVerification.getFingerprint();
2182 if (fingerprint == null) {
2183 return false;
2184 }
2185 final FingerprintStatus status =
2186 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2187 return status != null && status.isVerified();
2188 }
2189
2190 public boolean addMedia(final Media media) {
2191 final Set<Media> currentMedia = getMedia();
2192 if (currentMedia.contains(media)) {
2193 throw new IllegalStateException(String.format("%s has already been proposed", media));
2194 }
2195 // TODO add state protection - can only add while ACCEPTED or so
2196 Log.d(Config.LOGTAG, "adding media: " + media);
2197 return webRTCWrapper.addTrack(media);
2198 }
2199
2200 public synchronized void acceptCall() {
2201 switch (this.state) {
2202 case PROPOSED -> {
2203 cancelRingingTimeout();
2204 acceptCallFromProposed();
2205 }
2206 case SESSION_INITIALIZED -> {
2207 cancelRingingTimeout();
2208 acceptCallFromSessionInitialized();
2209 }
2210 case ACCEPTED ->
2211 Log.w(
2212 Config.LOGTAG,
2213 id.account.getJid().asBareJid()
2214 + ": the call has already been accepted with another client."
2215 + " UI was just lagging behind");
2216 case PROCEED, SESSION_ACCEPTED ->
2217 Log.w(
2218 Config.LOGTAG,
2219 id.account.getJid().asBareJid()
2220 + ": the call has already been accepted. user probably double"
2221 + " tapped the UI");
2222 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2223 }
2224 }
2225
2226 public synchronized void rejectCall() {
2227 if (isTerminated()) {
2228 Log.w(
2229 Config.LOGTAG,
2230 id.account.getJid().asBareJid()
2231 + ": received rejectCall() when session has already been terminated."
2232 + " nothing to do");
2233 return;
2234 }
2235 switch (this.state) {
2236 case PROPOSED -> rejectCallFromProposed();
2237 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2238 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2239 }
2240 }
2241
2242 public synchronized void integrationFailure() {
2243 final var state = getState();
2244 if (state == State.PROPOSED) {
2245 Log.e(
2246 Config.LOGTAG,
2247 id.account.getJid().asBareJid()
2248 + ": failed call integration in state proposed");
2249 rejectCallFromProposed();
2250 } else if (state == State.SESSION_INITIALIZED) {
2251 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2252 this.webRTCWrapper.close();
2253 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2254 } else {
2255 throw new IllegalStateException(
2256 String.format("Can not fail integration in state %s", state));
2257 }
2258 }
2259
2260 public synchronized void endCall() {
2261 if (isTerminated()) {
2262 Log.w(
2263 Config.LOGTAG,
2264 id.account.getJid().asBareJid()
2265 + ": received endCall() when session has already been terminated."
2266 + " nothing to do");
2267 return;
2268 }
2269 if (isInState(State.PROPOSED) && isResponder()) {
2270 rejectCallFromProposed();
2271 return;
2272 }
2273 if (isInState(State.PROCEED)) {
2274 if (isInitiator()) {
2275 retractFromProceed();
2276 } else {
2277 rejectCallFromProceed();
2278 }
2279 return;
2280 }
2281 if (isInitiator()
2282 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2283 this.webRTCWrapper.close();
2284 sendSessionTerminate(Reason.CANCEL);
2285 return;
2286 }
2287 if (isInState(State.SESSION_INITIALIZED)) {
2288 rejectCallFromSessionInitiate();
2289 return;
2290 }
2291 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2292 this.webRTCWrapper.close();
2293 sendSessionTerminate(Reason.SUCCESS);
2294 return;
2295 }
2296 if (isInState(
2297 State.TERMINATED_APPLICATION_FAILURE,
2298 State.TERMINATED_CONNECTIVITY_ERROR,
2299 State.TERMINATED_DECLINED_OR_BUSY)) {
2300 Log.d(
2301 Config.LOGTAG,
2302 "ignoring request to end call because already in state " + this.state);
2303 return;
2304 }
2305 throw new IllegalStateException(
2306 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2307 }
2308
2309 private void retractFromProceed() {
2310 Log.d(Config.LOGTAG, "retract from proceed");
2311 this.sendJingleMessage("retract");
2312 closeTransitionLogFinish(State.RETRACTED_RACED);
2313 }
2314
2315 private void closeTransitionLogFinish(final State state) {
2316 this.webRTCWrapper.close();
2317 transitionOrThrow(state);
2318 writeLogMessage(state);
2319 finish();
2320 }
2321
2322 private void setupWebRTC(
2323 final Set<Media> media,
2324 final Collection<PeerConnection.IceServer> iceServers,
2325 final boolean trickle)
2326 throws WebRTCWrapper.InitializationException {
2327 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2328 this.webRTCWrapper.setup(this.xmppConnectionService);
2329 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2330 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2331 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2332 }
2333
2334 private void acceptCallFromProposed() {
2335 transitionOrThrow(State.PROCEED);
2336 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2337 this.callIntegration.startAudioRouting();
2338 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2339 this.sendJingleMessage("proceed");
2340 }
2341
2342 private void rejectCallFromProposed() {
2343 transitionOrThrow(State.REJECTED);
2344 writeLogMessageMissed();
2345 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2346 this.sendJingleMessage("reject");
2347 finish();
2348 }
2349
2350 private void rejectCallFromProceed() {
2351 this.sendJingleMessage("reject");
2352 closeTransitionLogFinish(State.REJECTED_RACED);
2353 }
2354
2355 private void rejectCallFromSessionInitiate() {
2356 webRTCWrapper.close();
2357 sendSessionTerminate(Reason.DECLINE);
2358 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2359 }
2360
2361 private void sendJingleMessage(final String action) {
2362 sendJingleMessage(action, id.with);
2363 }
2364
2365 private void sendJingleMessage(final String action, final Jid to) {
2366 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2367 messagePacket.setType(
2368 im.conversations.android.xmpp.model.stanza.Message.Type
2369 .CHAT); // we want to carbon copy those
2370 messagePacket.setTo(to);
2371 final Element intent =
2372 messagePacket
2373 .addChild(action, Namespace.JINGLE_MESSAGE)
2374 .setAttribute("id", id.sessionId);
2375 if ("proceed".equals(action)) {
2376 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2377 if (isOmemoEnabled()) {
2378 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2379 final Element device =
2380 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2381 device.setAttribute("id", deviceId);
2382 }
2383 }
2384 messagePacket.addChild("store", "urn:xmpp:hints");
2385 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2386 }
2387
2388 private void sendJingleMessageFinish(final Reason reason) {
2389 final var account = id.getAccount();
2390 final var messagePacket =
2391 xmppConnectionService
2392 .getMessageGenerator()
2393 .sessionFinish(id.with, id.sessionId, reason);
2394 xmppConnectionService.sendMessagePacket(account, messagePacket);
2395 }
2396
2397 private boolean isOmemoEnabled() {
2398 final Conversational conversational = message.getConversation();
2399 if (conversational instanceof Conversation) {
2400 return ((Conversation) conversational).getNextEncryption()
2401 == Message.ENCRYPTION_AXOLOTL;
2402 }
2403 return false;
2404 }
2405
2406 private void acceptCallFromSessionInitialized() {
2407 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2408 this.callIntegration.startAudioRouting();
2409 sendSessionAccept();
2410 }
2411
2412 @Override
2413 protected synchronized boolean transition(final State target, final Runnable runnable) {
2414 if (super.transition(target, runnable)) {
2415 updateEndUserState();
2416 updateOngoingCallNotification();
2417 return true;
2418 } else {
2419 return false;
2420 }
2421 }
2422
2423 @Override
2424 public void onIceCandidate(final IceCandidate iceCandidate) {
2425 final RtpContentMap rtpContentMap =
2426 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2427 final IceUdpTransportInfo.Credentials credentials;
2428 try {
2429 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2430 } catch (final IllegalArgumentException e) {
2431 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2432 return;
2433 }
2434 final String uFrag = credentials.ufrag;
2435 final IceUdpTransportInfo.Candidate candidate =
2436 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2437 if (candidate == null) {
2438 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2439 return;
2440 }
2441 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2442 sendTransportInfo(iceCandidate.sdpMid, candidate);
2443 }
2444
2445 @Override
2446 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2447 Log.d(
2448 Config.LOGTAG,
2449 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2450 this.stateHistory.add(newState);
2451 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2452 this.sessionDuration.start();
2453 updateOngoingCallNotification();
2454 } else if (this.sessionDuration.isRunning()) {
2455 this.sessionDuration.stop();
2456 updateOngoingCallNotification();
2457 }
2458
2459 final boolean neverConnected =
2460 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2461
2462 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2463 if (neverConnected) {
2464 if (isTerminated()) {
2465 Log.d(
2466 Config.LOGTAG,
2467 id.account.getJid().asBareJid()
2468 + ": not sending session-terminate after connectivity error"
2469 + " because session is already in state "
2470 + this.state);
2471 return;
2472 }
2473 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2474 return;
2475 } else {
2476 this.restartIce();
2477 }
2478 }
2479 updateEndUserState();
2480 }
2481
2482 private void restartIce() {
2483 this.stateHistory.clear();
2484 this.webRTCWrapper.restartIceAsync();
2485 }
2486
2487 @Override
2488 public void onRenegotiationNeeded() {
2489 this.webRTCWrapper.execute(this::renegotiate);
2490 }
2491
2492 private synchronized void renegotiate() {
2493 final SessionDescription sessionDescription;
2494 try {
2495 sessionDescription = setLocalSessionDescription();
2496 } catch (final Exception e) {
2497 final Throwable cause = Throwables.getRootCause(e);
2498 webRTCWrapper.close();
2499 if (isTerminated()) {
2500 Log.d(
2501 Config.LOGTAG,
2502 "failed to renegotiate. session was already terminated",
2503 cause);
2504 return;
2505 }
2506 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2507 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2508 return;
2509 }
2510 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2511 final RtpContentMap currentContentMap = getLocalContentMap();
2512 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2513 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2514
2515 Log.d(
2516 Config.LOGTAG,
2517 id.getAccount().getJid().asBareJid()
2518 + ": renegotiate. iceRestart="
2519 + iceRestart
2520 + " content id diff="
2521 + diff);
2522
2523 if (diff.hasModifications() && iceRestart) {
2524 webRTCWrapper.close();
2525 sendSessionTerminate(
2526 Reason.FAILED_APPLICATION,
2527 "WebRTC unexpectedly tried to modify content and transport at once");
2528 return;
2529 }
2530
2531 if (iceRestart) {
2532 initiateIceRestart(rtpContentMap);
2533 return;
2534 } else if (diff.isEmpty()) {
2535 Log.d(
2536 Config.LOGTAG,
2537 "renegotiation. nothing to do. SignalingState="
2538 + this.webRTCWrapper.getSignalingState());
2539 }
2540
2541 if (diff.added.isEmpty()) {
2542 return;
2543 }
2544 modifyLocalContentMap(rtpContentMap);
2545 sendContentAdd(rtpContentMap, diff.added);
2546 }
2547
2548 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2549 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2550 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2551 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2552 iq.setTo(id.with);
2553 xmppConnectionService.sendIqPacket(
2554 id.account,
2555 iq,
2556 (response) -> {
2557 if (response.getType() == Iq.Type.RESULT) {
2558 Log.d(Config.LOGTAG, "received success to our ice restart");
2559 setLocalContentMap(rtpContentMap);
2560 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2561 return;
2562 }
2563 if (response.getType() == Iq.Type.ERROR) {
2564 if (isTieBreak(response)) {
2565 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2566 return;
2567 }
2568 handleIqErrorResponse(response);
2569 }
2570 if (response.getType() == Iq.Type.TIMEOUT) {
2571 handleIqTimeoutResponse(response);
2572 }
2573 });
2574 }
2575
2576 private boolean isTieBreak(final Iq response) {
2577 final Element error = response.findChild("error");
2578 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2579 }
2580
2581 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2582 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2583 this.outgoingContentAdd = contentAdd;
2584 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2585 prepareOutgoingContentMap(contentAdd);
2586 Futures.addCallback(
2587 outgoingContentMapFuture,
2588 new FutureCallback<>() {
2589 @Override
2590 public void onSuccess(final RtpContentMap outgoingContentMap) {
2591 sendContentAdd(outgoingContentMap);
2592 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2593 }
2594
2595 @Override
2596 public void onFailure(@NonNull Throwable throwable) {
2597 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2598 }
2599 },
2600 MoreExecutors.directExecutor());
2601 }
2602
2603 private void sendContentAdd(final RtpContentMap contentAdd) {
2604
2605 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2606 iq.setTo(id.with);
2607 xmppConnectionService.sendIqPacket(
2608 id.account,
2609 iq,
2610 (response) -> {
2611 if (response.getType() == Iq.Type.RESULT) {
2612 Log.d(
2613 Config.LOGTAG,
2614 id.getAccount().getJid().asBareJid()
2615 + ": received ACK to our content-add");
2616 return;
2617 }
2618 if (response.getType() == Iq.Type.ERROR) {
2619 if (isTieBreak(response)) {
2620 this.outgoingContentAdd = null;
2621 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2622 return;
2623 }
2624 handleIqErrorResponse(response);
2625 }
2626 if (response.getType() == Iq.Type.TIMEOUT) {
2627 handleIqTimeoutResponse(response);
2628 }
2629 });
2630 }
2631
2632 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2633 if (isInitiator()) {
2634 this.initiatorRtpContentMap = rtpContentMap;
2635 } else {
2636 this.responderRtpContentMap = rtpContentMap;
2637 }
2638 }
2639
2640 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2641 if (isInitiator()) {
2642 this.responderRtpContentMap = rtpContentMap;
2643 } else {
2644 this.initiatorRtpContentMap = rtpContentMap;
2645 }
2646 }
2647
2648 // this method is to be used for content map modifications that modify media
2649 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2650 final RtpContentMap activeContents = rtpContentMap.activeContents();
2651 setLocalContentMap(activeContents);
2652 this.callIntegration.setAudioDeviceWhenAvailable(
2653 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2654 updateEndUserState();
2655 }
2656
2657 private SessionDescription setLocalSessionDescription()
2658 throws ExecutionException, InterruptedException {
2659 final org.webrtc.SessionDescription sessionDescription =
2660 this.webRTCWrapper.setLocalDescription(false).get();
2661 return SessionDescription.parse(sessionDescription.description);
2662 }
2663
2664 private void closeWebRTCSessionAfterFailedConnection() {
2665 this.webRTCWrapper.close();
2666 synchronized (this) {
2667 if (isTerminated()) {
2668 Log.d(
2669 Config.LOGTAG,
2670 id.account.getJid().asBareJid()
2671 + ": no need to send session-terminate after failed connection."
2672 + " Other party already did");
2673 return;
2674 }
2675 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2676 }
2677 }
2678
2679 public boolean zeroDuration() {
2680 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2681 }
2682
2683 public long getCallDuration() {
2684 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2685 }
2686
2687 @Override
2688 public CallIntegration getCallIntegration() {
2689 return this.callIntegration;
2690 }
2691
2692 public boolean isMicrophoneEnabled() {
2693 return webRTCWrapper.isMicrophoneEnabled();
2694 }
2695
2696 public boolean setMicrophoneEnabled(final boolean enabled) {
2697 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2698 }
2699
2700 public boolean isVideoEnabled() {
2701 return webRTCWrapper.isVideoEnabled();
2702 }
2703
2704 public void setVideoEnabled(final boolean enabled) {
2705 webRTCWrapper.setVideoEnabled(enabled);
2706 }
2707
2708 public boolean isCameraSwitchable() {
2709 return webRTCWrapper.isCameraSwitchable();
2710 }
2711
2712 public boolean isFrontCamera() {
2713 return webRTCWrapper.isFrontCamera();
2714 }
2715
2716 public ListenableFuture<Boolean> switchCamera() {
2717 return webRTCWrapper.switchCamera();
2718 }
2719
2720 @Override
2721 public synchronized void onCallIntegrationShowIncomingCallUi() {
2722 if (isTerminated()) {
2723 // there might be race conditions with the call integration service invoking this
2724 // callback when the rtp session has already ended.
2725 Log.w(
2726 Config.LOGTAG,
2727 "CallIntegration requested incoming call UI but session was already"
2728 + " terminated");
2729 return;
2730 }
2731 // TODO apparently this can be called too early as well?
2732 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2733 }
2734
2735 @Override
2736 public void onCallIntegrationDisconnect() {
2737 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2738 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2739 rejectCall();
2740 } else {
2741 endCall();
2742 }
2743 }
2744
2745 @Override
2746 public void onCallIntegrationReject() {
2747 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2748 try {
2749 rejectCall();
2750 } catch (final IllegalStateException e) {
2751 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2752 }
2753 }
2754
2755 @Override
2756 public void onCallIntegrationAnswer() {
2757 // we need to start the UI to a) show it and b) be able to ask for permissions
2758 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2759 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2760 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toString());
2761 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString());
2762 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2763 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2764 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2765 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2766 xmppConnectionService.startActivity(intent);
2767 }
2768
2769 @Override
2770 public void onCallIntegrationSilence() {
2771 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2772 }
2773
2774 @Override
2775 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2776 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2777 // that was previous disabled by the user. A proper implementation would probably be to
2778 // track user choice and enable the microphone with a userEnabled() &&
2779 // callIntegration.isMicrophoneEnabled() condition
2780 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2781 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2782 }
2783
2784 @Override
2785 public void onAudioDeviceChanged(
2786 final CallIntegration.AudioDevice selectedAudioDevice,
2787 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2788 Log.d(
2789 Config.LOGTAG,
2790 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2791 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2792 selectedAudioDevice, availableAudioDevices);
2793 }
2794
2795 private void updateEndUserState() {
2796 final RtpEndUserState endUserState = getEndUserState();
2797 this.updateCallIntegrationState();
2798 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2799 id.account, id.with, id.sessionId, endUserState);
2800 }
2801
2802 private void updateOngoingCallNotification() {
2803 final State state = this.state;
2804 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2805 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2806 && isResponder()) {
2807 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2808 xmppConnectionService.removeOngoingCall();
2809 return;
2810 }
2811 final boolean reconnecting;
2812 if (state == State.SESSION_ACCEPTED) {
2813 reconnecting =
2814 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2815 } else {
2816 reconnecting = false;
2817 }
2818 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2819 } else {
2820 xmppConnectionService.removeOngoingCall();
2821 }
2822 }
2823
2824 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2825 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2826 final Iq request = new Iq(Iq.Type.GET);
2827 request.setTo(id.account.getDomain());
2828 request.addExtension(new Services());
2829 xmppConnectionService.sendIqPacket(
2830 id.account,
2831 request,
2832 (response) -> {
2833 final var iceServers = IceServers.parse(response);
2834 if (iceServers.isEmpty()) {
2835 Log.w(
2836 Config.LOGTAG,
2837 id.account.getJid().asBareJid()
2838 + ": no ICE server found "
2839 + response);
2840 }
2841 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2842 });
2843 } else {
2844 Log.w(
2845 Config.LOGTAG,
2846 id.account.getJid().asBareJid() + ": has no external service discovery");
2847 onIceServersDiscovered.onIceServersDiscovered(Collections.emptySet());
2848 }
2849 }
2850
2851 @Override
2852 protected void terminateTransport() {
2853 this.webRTCWrapper.close();
2854 }
2855
2856 @Override
2857 protected void finish() {
2858 if (isTerminated()) {
2859 this.cancelRingingTimeout();
2860 this.callIntegration.verifyDisconnected();
2861 this.webRTCWrapper.verifyClosed();
2862 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2863 super.finish();
2864 } else {
2865 throw new IllegalStateException(
2866 String.format("Unable to call finish from %s", this.state));
2867 }
2868 }
2869
2870 private void writeLogMessage(final State state) {
2871 final long duration = getCallDuration();
2872 if (state == State.TERMINATED_SUCCESS
2873 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2874 writeLogMessageSuccess(duration);
2875 } else {
2876 writeLogMessageMissed();
2877 }
2878 }
2879
2880 private void writeLogMessageSuccess(final long duration) {
2881 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2882 this.writeMessage();
2883 }
2884
2885 private void writeLogMessageMissed() {
2886 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2887 this.writeMessage();
2888 }
2889
2890 private void writeMessage() {
2891 final Conversational conversational = message.getConversation();
2892 if (conversational instanceof Conversation) {
2893 ((Conversation) conversational).add(this.message);
2894 xmppConnectionService.createMessageAsync(message);
2895 xmppConnectionService.updateConversationUi();
2896 } else {
2897 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2898 }
2899 }
2900
2901 public Optional<VideoTrack> getLocalVideoTrack() {
2902 return webRTCWrapper.getLocalVideoTrack();
2903 }
2904
2905 public Optional<VideoTrack> getRemoteVideoTrack() {
2906 return webRTCWrapper.getRemoteVideoTrack();
2907 }
2908
2909 public EglBase.Context getEglBaseContext() {
2910 return webRTCWrapper.getEglBaseContext();
2911 }
2912
2913 void setProposedMedia(final Set<Media> media) {
2914 this.proposedMedia = media;
2915 this.callIntegration.setVideoState(
2916 Media.audioOnly(media)
2917 ? VideoProfile.STATE_AUDIO_ONLY
2918 : VideoProfile.STATE_BIDIRECTIONAL);
2919 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2920 }
2921
2922 public void fireStateUpdate() {
2923 final RtpEndUserState endUserState = getEndUserState();
2924 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2925 id.account, id.with, id.sessionId, endUserState);
2926 }
2927
2928 public boolean isSwitchToVideoAvailable() {
2929 final boolean prerequisite =
2930 Media.audioOnly(getMedia())
2931 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2932 .contains(getEndUserState());
2933 return prerequisite && remoteHasVideoFeature();
2934 }
2935
2936 private boolean remoteHasVideoFeature() {
2937 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2938 }
2939
2940 private boolean remoteHasSdpOfferAnswer() {
2941 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2942 }
2943
2944 @Override
2945 public Account getAccount() {
2946 return id.account;
2947 }
2948
2949 @Override
2950 public Jid getWith() {
2951 return id.with;
2952 }
2953
2954 @Override
2955 public String getSessionId() {
2956 return id.sessionId;
2957 }
2958
2959 private interface OnIceServersDiscovered {
2960 void onIceServersDiscovered(Collection<PeerConnection.IceServer> iceServers);
2961 }
2962}