1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.ui.RtpSessionActivity;
39import eu.siacs.conversations.xml.Element;
40import eu.siacs.conversations.xml.Namespace;
41import eu.siacs.conversations.xmpp.Jid;
42import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
44import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
45import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import eu.siacs.conversations.xmpp.stanzas.IqPacket;
51import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
52
53import org.webrtc.EglBase;
54import org.webrtc.IceCandidate;
55import org.webrtc.PeerConnection;
56import org.webrtc.VideoTrack;
57
58import java.util.Arrays;
59import java.util.Collection;
60import java.util.Collections;
61import java.util.LinkedList;
62import java.util.List;
63import java.util.Map;
64import java.util.Queue;
65import java.util.Set;
66import java.util.concurrent.ExecutionException;
67import java.util.concurrent.ScheduledFuture;
68import java.util.concurrent.TimeUnit;
69
70public class JingleRtpConnection extends AbstractJingleConnection
71 implements WebRTCWrapper.EventCallback, CallIntegration.Callback {
72
73 public static final List<State> STATES_SHOWING_ONGOING_CALL =
74 Arrays.asList(
75 State.PROPOSED,
76 State.PROCEED,
77 State.SESSION_INITIALIZED_PRE_APPROVED,
78 State.SESSION_ACCEPTED);
79 private static final long BUSY_TIME_OUT = 30;
80
81 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
82 private final Queue<
83 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
84 pendingIceCandidates = new LinkedList<>();
85 private final OmemoVerification omemoVerification = new OmemoVerification();
86 private final CallIntegration callIntegration;
87 private final Message message;
88
89 private Set<Media> proposedMedia;
90 private RtpContentMap initiatorRtpContentMap;
91 private RtpContentMap responderRtpContentMap;
92 private RtpContentMap incomingContentAdd;
93 private RtpContentMap outgoingContentAdd;
94 private IceUdpTransportInfo.Setup peerDtlsSetup;
95 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
96 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
97 private ScheduledFuture<?> ringingTimeoutFuture;
98
99 JingleRtpConnection(
100 final JingleConnectionManager jingleConnectionManager,
101 final Id id,
102 final Jid initiator) {
103 this(
104 jingleConnectionManager,
105 id,
106 initiator,
107 new CallIntegration(
108 jingleConnectionManager
109 .getXmppConnectionService()
110 .getApplicationContext()));
111 this.callIntegration.setAddress(
112 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
113 this.callIntegration.setInitialized();
114 }
115
116 JingleRtpConnection(
117 final JingleConnectionManager jingleConnectionManager,
118 final Id id,
119 final Jid initiator,
120 final CallIntegration callIntegration) {
121 super(jingleConnectionManager, id, initiator);
122 final Conversation conversation =
123 jingleConnectionManager
124 .getXmppConnectionService()
125 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
126 this.message =
127 new Message(
128 conversation,
129 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
130 Message.TYPE_RTP_SESSION,
131 id.sessionId);
132 this.callIntegration = callIntegration;
133 this.callIntegration.setCallback(this);
134 }
135
136 @Override
137 synchronized void deliverPacket(final JinglePacket jinglePacket) {
138 switch (jinglePacket.getAction()) {
139 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
140 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
141 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
142 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
143 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
144 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
145 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
146 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
147 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
148 default -> {
149 respondOk(jinglePacket);
150 Log.d(
151 Config.LOGTAG,
152 String.format(
153 "%s: received unhandled jingle action %s",
154 id.account.getJid().asBareJid(), jinglePacket.getAction()));
155 }
156 }
157 }
158
159 @Override
160 synchronized void notifyRebound() {
161 if (isTerminated()) {
162 return;
163 }
164 webRTCWrapper.close();
165 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
166 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
167 }
168 if (isInState(
169 State.SESSION_INITIALIZED,
170 State.SESSION_INITIALIZED_PRE_APPROVED,
171 State.SESSION_ACCEPTED)) {
172 // we might have already changed resources (full jid) at this point; so this might not
173 // even reach the other party
174 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
175 } else {
176 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
177 finish();
178 }
179 }
180
181 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
182 respondOk(jinglePacket);
183 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
184 final State previous = this.state;
185 Log.d(
186 Config.LOGTAG,
187 id.account.getJid().asBareJid()
188 + ": received session terminate reason="
189 + wrapper.reason
190 + "("
191 + Strings.nullToEmpty(wrapper.text)
192 + ") while in state "
193 + previous);
194 if (TERMINATED.contains(previous)) {
195 Log.d(
196 Config.LOGTAG,
197 id.account.getJid().asBareJid()
198 + ": ignoring session terminate because already in "
199 + previous);
200 return;
201 }
202 webRTCWrapper.close();
203 final State target = reasonToState(wrapper.reason);
204 transitionOrThrow(target);
205 writeLogMessage(target);
206 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
207 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
208 }
209 finish();
210 }
211
212 private void receiveTransportInfo(final JinglePacket jinglePacket) {
213 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
214 // INITIALIZED only after transport-info has been received
215 if (isInState(
216 State.NULL,
217 State.PROCEED,
218 State.SESSION_INITIALIZED,
219 State.SESSION_INITIALIZED_PRE_APPROVED,
220 State.SESSION_ACCEPTED)) {
221 final RtpContentMap contentMap;
222 try {
223 contentMap = RtpContentMap.of(jinglePacket);
224 } catch (final IllegalArgumentException | NullPointerException e) {
225 Log.d(
226 Config.LOGTAG,
227 id.account.getJid().asBareJid()
228 + ": improperly formatted contents; ignoring",
229 e);
230 respondOk(jinglePacket);
231 return;
232 }
233 receiveTransportInfo(jinglePacket, contentMap);
234 } else {
235 if (isTerminated()) {
236 respondOk(jinglePacket);
237 Log.d(
238 Config.LOGTAG,
239 id.account.getJid().asBareJid()
240 + ": ignoring out-of-order transport info; we where already terminated");
241 } else {
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": received transport info while in state="
246 + this.state);
247 terminateWithOutOfOrder(jinglePacket);
248 }
249 }
250 }
251
252 private void receiveTransportInfo(
253 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
254 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
255 candidates = contentMap.contents.entrySet();
256 final RtpContentMap remote = getRemoteContentMap();
257 final Set<String> remoteContentIds =
258 remote == null ? Collections.emptySet() : remote.contents.keySet();
259 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
260 Log.d(
261 Config.LOGTAG,
262 "received transport-info for unknown contents "
263 + contentMap.contents.keySet()
264 + " (known: "
265 + remoteContentIds
266 + ")");
267 respondOk(jinglePacket);
268 pendingIceCandidates.addAll(candidates);
269 return;
270 }
271 if (this.state != State.SESSION_ACCEPTED) {
272 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
273 respondOk(jinglePacket);
274 pendingIceCandidates.addAll(candidates);
275 return;
276 }
277 // zero candidates + modified credentials are an ICE restart offer
278 if (checkForIceRestart(jinglePacket, contentMap)) {
279 return;
280 }
281 respondOk(jinglePacket);
282 try {
283 processCandidates(candidates);
284 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
285 Log.w(
286 Config.LOGTAG,
287 id.account.getJid().asBareJid()
288 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
289 }
290 }
291
292 private void receiveContentAdd(final JinglePacket jinglePacket) {
293 final RtpContentMap modification;
294 try {
295 modification = RtpContentMap.of(jinglePacket);
296 modification.requireContentDescriptions();
297 } catch (final RuntimeException e) {
298 Log.d(
299 Config.LOGTAG,
300 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
301 Throwables.getRootCause(e));
302 respondOk(jinglePacket);
303 webRTCWrapper.close();
304 sendSessionTerminate(Reason.of(e), e.getMessage());
305 return;
306 }
307 if (isInState(State.SESSION_ACCEPTED)) {
308 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
309 final ListenableFuture<RtpContentMap> future =
310 receiveRtpContentMap(
311 modification,
312 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
313 Futures.addCallback(
314 future,
315 new FutureCallback<>() {
316 @Override
317 public void onSuccess(final RtpContentMap rtpContentMap) {
318 receiveContentAdd(jinglePacket, rtpContentMap);
319 }
320
321 @Override
322 public void onFailure(@NonNull Throwable throwable) {
323 respondOk(jinglePacket);
324 final Throwable rootCause = Throwables.getRootCause(throwable);
325 Log.d(
326 Config.LOGTAG,
327 id.account.getJid().asBareJid()
328 + ": improperly formatted contents in content-add",
329 throwable);
330 webRTCWrapper.close();
331 sendSessionTerminate(
332 Reason.ofThrowable(rootCause), rootCause.getMessage());
333 }
334 },
335 MoreExecutors.directExecutor());
336 } else {
337 terminateWithOutOfOrder(jinglePacket);
338 }
339 }
340
341 private void receiveContentAdd(
342 final JinglePacket jinglePacket, final RtpContentMap modification) {
343 final RtpContentMap remote = getRemoteContentMap();
344 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
345 respondOk(jinglePacket);
346 this.webRTCWrapper.close();
347 sendSessionTerminate(
348 Reason.FAILED_APPLICATION,
349 String.format(
350 "contents with names %s already exists",
351 Joiner.on(", ").join(modification.getNames())));
352 return;
353 }
354 final ContentAddition contentAddition =
355 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
356
357 final RtpContentMap outgoing = this.outgoingContentAdd;
358 final Set<ContentAddition.Summary> outgoingContentAddSummary =
359 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
360
361 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
362 if (isInitiator()) {
363 Log.d(
364 Config.LOGTAG,
365 id.getAccount().getJid().asBareJid()
366 + ": respond with tie break to matching content-add offer");
367 respondWithTieBreak(jinglePacket);
368 } else {
369 Log.d(
370 Config.LOGTAG,
371 id.getAccount().getJid().asBareJid()
372 + ": automatically accept matching content-add offer");
373 acceptContentAdd(contentAddition.summary, modification);
374 }
375 return;
376 }
377
378 // once we can display multiple video tracks we can be more loose with this condition
379 // theoretically it should also be fine to automatically accept audio only contents
380 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
381 Log.d(
382 Config.LOGTAG,
383 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
384 this.incomingContentAdd = modification;
385 respondOk(jinglePacket);
386 updateEndUserState();
387 } else {
388 respondOk(jinglePacket);
389 // TODO do we want to add a reason?
390 rejectContentAdd(modification);
391 }
392 }
393
394 private void receiveContentAccept(final JinglePacket jinglePacket) {
395 final RtpContentMap receivedContentAccept;
396 try {
397 receivedContentAccept = RtpContentMap.of(jinglePacket);
398 receivedContentAccept.requireContentDescriptions();
399 } catch (final RuntimeException e) {
400 Log.d(
401 Config.LOGTAG,
402 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
403 Throwables.getRootCause(e));
404 respondOk(jinglePacket);
405 webRTCWrapper.close();
406 sendSessionTerminate(Reason.of(e), e.getMessage());
407 return;
408 }
409
410 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
411 if (outgoingContentAdd == null) {
412 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
413 terminateWithOutOfOrder(jinglePacket);
414 return;
415 }
416 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
417 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
418 this.outgoingContentAdd = null;
419 respondOk(jinglePacket);
420 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
421 final ListenableFuture<RtpContentMap> future =
422 receiveRtpContentMap(
423 receivedContentAccept,
424 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
425 Futures.addCallback(
426 future,
427 new FutureCallback<>() {
428 @Override
429 public void onSuccess(final RtpContentMap result) {
430 receiveContentAccept(result);
431 }
432
433 @Override
434 public void onFailure(@NonNull final Throwable throwable) {
435 webRTCWrapper.close();
436 sendSessionTerminate(
437 Reason.ofThrowable(throwable), throwable.getMessage());
438 }
439 },
440 MoreExecutors.directExecutor());
441 } else {
442 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
443 terminateWithOutOfOrder(jinglePacket);
444 }
445 }
446
447 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
448 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
449 final RtpContentMap modifiedContentMap =
450 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
451
452 setRemoteContentMap(modifiedContentMap);
453
454 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
455
456 final org.webrtc.SessionDescription sdp =
457 new org.webrtc.SessionDescription(
458 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
459
460 try {
461 this.webRTCWrapper.setRemoteDescription(sdp).get();
462 } catch (final Exception e) {
463 final Throwable cause = Throwables.getRootCause(e);
464 Log.d(
465 Config.LOGTAG,
466 id.getAccount().getJid().asBareJid()
467 + ": unable to set remote description after receiving content-accept",
468 cause);
469 webRTCWrapper.close();
470 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
471 return;
472 }
473 Log.d(
474 Config.LOGTAG,
475 id.getAccount().getJid().asBareJid()
476 + ": remote has accepted content-add "
477 + ContentAddition.summary(receivedContentAccept));
478 processCandidates(receivedContentAccept.contents.entrySet());
479 updateEndUserState();
480 }
481
482 private void receiveContentModify(final JinglePacket jinglePacket) {
483 if (this.state != State.SESSION_ACCEPTED) {
484 terminateWithOutOfOrder(jinglePacket);
485 return;
486 }
487 final Map<String, Content.Senders> modification =
488 Maps.transformEntries(
489 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
490 final boolean isInitiator = isInitiator();
491 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
492 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
493 final Set<String> currentOutgoingMediaIds =
494 currentOutgoing == null
495 ? Collections.emptySet()
496 : currentOutgoing.contents.keySet();
497 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
498 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
499 respondOk(jinglePacket);
500 final RtpContentMap modifiedContentMap;
501 try {
502 modifiedContentMap =
503 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
504 } catch (final IllegalArgumentException e) {
505 webRTCWrapper.close();
506 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
507 return;
508 }
509 this.outgoingContentAdd = modifiedContentMap;
510 Log.d(
511 Config.LOGTAG,
512 id.account.getJid().asBareJid()
513 + ": processed content-modification for pending content-add");
514 } else if (remoteContentMap != null
515 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
516 respondOk(jinglePacket);
517 final RtpContentMap modifiedRemoteContentMap;
518 try {
519 modifiedRemoteContentMap =
520 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
521 } catch (final IllegalArgumentException e) {
522 webRTCWrapper.close();
523 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
524 return;
525 }
526 final SessionDescription offer;
527 try {
528 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
529 } catch (final IllegalArgumentException | NullPointerException e) {
530 Log.d(
531 Config.LOGTAG,
532 id.getAccount().getJid().asBareJid()
533 + ": unable convert offer from content-modify to SDP",
534 e);
535 webRTCWrapper.close();
536 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
537 return;
538 }
539 Log.d(
540 Config.LOGTAG,
541 id.account.getJid().asBareJid() + ": auto accepting content-modification");
542 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
543 } else {
544 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
545 respondWithItemNotFound(jinglePacket);
546 }
547 }
548
549 private void autoAcceptContentModify(
550 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
551 this.setRemoteContentMap(modifiedRemoteContentMap);
552 final org.webrtc.SessionDescription sdp =
553 new org.webrtc.SessionDescription(
554 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
555 try {
556 this.webRTCWrapper.setRemoteDescription(sdp).get();
557 // auto accept is only done when we already have tracks
558 final SessionDescription answer = setLocalSessionDescription();
559 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
560 modifyLocalContentMap(rtpContentMap);
561 // we do not need to send an answer but do we have to resend the candidates currently in
562 // SDP?
563 // resendCandidatesFromSdp(answer);
564 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
565 } catch (final Exception e) {
566 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.FAILED_APPLICATION);
569 }
570 }
571
572 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
573 final SessionDescription answer) {
574 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
575 new ImmutableMultimap.Builder<>();
576 for (final SessionDescription.Media media : answer.media) {
577 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
578 if (Strings.isNullOrEmpty(mid)) {
579 continue;
580 }
581 for (final String sdpCandidate : media.attributes.get("candidate")) {
582 final IceUdpTransportInfo.Candidate candidate =
583 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
584 if (candidate != null) {
585 candidateBuilder.put(mid, candidate);
586 }
587 }
588 }
589 return candidateBuilder.build();
590 }
591
592 private void receiveContentReject(final JinglePacket jinglePacket) {
593 final RtpContentMap receivedContentReject;
594 try {
595 receivedContentReject = RtpContentMap.of(jinglePacket);
596 } catch (final RuntimeException e) {
597 Log.d(
598 Config.LOGTAG,
599 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
600 Throwables.getRootCause(e));
601 respondOk(jinglePacket);
602 this.webRTCWrapper.close();
603 sendSessionTerminate(Reason.of(e), e.getMessage());
604 return;
605 }
606
607 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
608 if (outgoingContentAdd == null) {
609 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
610 terminateWithOutOfOrder(jinglePacket);
611 return;
612 }
613 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
614 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
615 this.outgoingContentAdd = null;
616 respondOk(jinglePacket);
617 Log.d(Config.LOGTAG, jinglePacket.toString());
618 receiveContentReject(ourSummary);
619 } else {
620 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
621 terminateWithOutOfOrder(jinglePacket);
622 }
623 }
624
625 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
626 try {
627 this.webRTCWrapper.removeTrack(Media.VIDEO);
628 final RtpContentMap localContentMap = customRollback();
629 modifyLocalContentMap(localContentMap);
630 } catch (final Exception e) {
631 final Throwable cause = Throwables.getRootCause(e);
632 Log.d(
633 Config.LOGTAG,
634 id.getAccount().getJid().asBareJid()
635 + ": unable to rollback local description after receiving content-reject",
636 cause);
637 webRTCWrapper.close();
638 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
639 return;
640 }
641 Log.d(
642 Config.LOGTAG,
643 id.getAccount().getJid().asBareJid()
644 + ": remote has rejected our content-add "
645 + summary);
646 }
647
648 private void receiveContentRemove(final JinglePacket jinglePacket) {
649 final RtpContentMap receivedContentRemove;
650 try {
651 receivedContentRemove = RtpContentMap.of(jinglePacket);
652 receivedContentRemove.requireContentDescriptions();
653 } catch (final RuntimeException e) {
654 Log.d(
655 Config.LOGTAG,
656 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
657 Throwables.getRootCause(e));
658 respondOk(jinglePacket);
659 this.webRTCWrapper.close();
660 sendSessionTerminate(Reason.of(e), e.getMessage());
661 return;
662 }
663 respondOk(jinglePacket);
664 receiveContentRemove(receivedContentRemove);
665 }
666
667 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
668 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
669 final Set<ContentAddition.Summary> contentAddSummary =
670 incomingContentAdd == null
671 ? Collections.emptySet()
672 : ContentAddition.summary(incomingContentAdd);
673 final Set<ContentAddition.Summary> removeSummary =
674 ContentAddition.summary(receivedContentRemove);
675 if (contentAddSummary.equals(removeSummary)) {
676 this.incomingContentAdd = null;
677 updateEndUserState();
678 } else {
679 webRTCWrapper.close();
680 sendSessionTerminate(
681 Reason.FAILED_APPLICATION,
682 String.format(
683 "%s only supports %s as a means to retract a not yet accepted %s",
684 BuildConfig.APP_NAME,
685 JinglePacket.Action.CONTENT_REMOVE,
686 JinglePacket.Action.CONTENT_ADD));
687 }
688 }
689
690 public synchronized void retractContentAdd() {
691 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
692 if (outgoingContentAdd == null) {
693 throw new IllegalStateException("Not outgoing content add");
694 }
695 try {
696 webRTCWrapper.removeTrack(Media.VIDEO);
697 final RtpContentMap localContentMap = customRollback();
698 modifyLocalContentMap(localContentMap);
699 } catch (final Exception e) {
700 final Throwable cause = Throwables.getRootCause(e);
701 Log.d(
702 Config.LOGTAG,
703 id.getAccount().getJid().asBareJid()
704 + ": unable to rollback local description after trying to retract content-add",
705 cause);
706 webRTCWrapper.close();
707 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
708 return;
709 }
710 this.outgoingContentAdd = null;
711 final JinglePacket retract =
712 outgoingContentAdd
713 .toStub()
714 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
715 this.send(retract);
716 Log.d(
717 Config.LOGTAG,
718 id.getAccount().getJid()
719 + ": retract content-add "
720 + ContentAddition.summary(outgoingContentAdd));
721 }
722
723 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
724 final SessionDescription sdp = setLocalSessionDescription();
725 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
726 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
727 this.webRTCWrapper
728 .setRemoteDescription(
729 new org.webrtc.SessionDescription(
730 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
731 .get();
732 return localRtpContentMap;
733 }
734
735 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
736 final RtpContentMap currentRemote = getRemoteContentMap();
737 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
738 if (diff.isEmpty()) {
739 throw new IllegalStateException(
740 "Unexpected rollback condition. No difference between local and remote");
741 }
742 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
743 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
744 final RtpContentMap nextRemote =
745 currentRemote.addContent(
746 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
747 return SessionDescription.of(nextRemote, isResponder());
748 }
749 throw new IllegalStateException(
750 "Unexpected rollback condition. Senders were not uniformly none");
751 }
752
753 public synchronized void acceptContentAdd(
754 @NonNull final Set<ContentAddition.Summary> contentAddition) {
755 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
756 if (incomingContentAdd == null) {
757 throw new IllegalStateException("No incoming content add");
758 }
759
760 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
761 this.incomingContentAdd = null;
762 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
763 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
764 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
765 Log.d(
766 Config.LOGTAG,
767 "content addition is receive only. we want to upgrade to 'both'");
768 final RtpContentMap modifiedSenders =
769 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
770 final JinglePacket proposedContentModification =
771 modifiedSenders
772 .toStub()
773 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
774 proposedContentModification.setTo(id.with);
775 xmppConnectionService.sendIqPacket(
776 id.account,
777 proposedContentModification,
778 (account, response) -> {
779 if (response.getType() == IqPacket.TYPE.RESULT) {
780 Log.d(
781 Config.LOGTAG,
782 id.account.getJid().asBareJid()
783 + ": remote has accepted our upgrade to senders=both");
784 acceptContentAdd(
785 ContentAddition.summary(modifiedSenders), modifiedSenders);
786 } else {
787 Log.d(
788 Config.LOGTAG,
789 id.account.getJid().asBareJid()
790 + ": remote has rejected our upgrade to senders=both");
791 acceptContentAdd(contentAddition, incomingContentAdd);
792 }
793 });
794 } else {
795 acceptContentAdd(contentAddition, incomingContentAdd);
796 }
797 } else {
798 throw new IllegalStateException(
799 "Accepted content add does not match pending content-add");
800 }
801 }
802
803 private void acceptContentAdd(
804 @NonNull final Set<ContentAddition.Summary> contentAddition,
805 final RtpContentMap incomingContentAdd) {
806 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
807 final RtpContentMap modifiedContentMap =
808 getRemoteContentMap().addContent(incomingContentAdd, setup);
809 this.setRemoteContentMap(modifiedContentMap);
810
811 final SessionDescription offer;
812 try {
813 offer = SessionDescription.of(modifiedContentMap, isResponder());
814 } catch (final IllegalArgumentException | NullPointerException e) {
815 Log.d(
816 Config.LOGTAG,
817 id.getAccount().getJid().asBareJid()
818 + ": unable convert offer from content-add to SDP",
819 e);
820 webRTCWrapper.close();
821 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
822 return;
823 }
824 this.incomingContentAdd = null;
825 acceptContentAdd(contentAddition, offer);
826 }
827
828 private void acceptContentAdd(
829 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
830 final org.webrtc.SessionDescription sdp =
831 new org.webrtc.SessionDescription(
832 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
833 try {
834 this.webRTCWrapper.setRemoteDescription(sdp).get();
835
836 // TODO add tracks for 'media' where contentAddition.senders matches
837
838 // TODO if senders.sending(isInitiator())
839
840 this.webRTCWrapper.addTrack(Media.VIDEO);
841
842 // TODO add additional transceivers for recv only cases
843
844 final SessionDescription answer = setLocalSessionDescription();
845 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
846
847 final RtpContentMap contentAcceptMap =
848 rtpContentMap.toContentModification(
849 Collections2.transform(contentAddition, ca -> ca.name));
850
851 Log.d(
852 Config.LOGTAG,
853 id.getAccount().getJid().asBareJid()
854 + ": sending content-accept "
855 + ContentAddition.summary(contentAcceptMap));
856
857 addIceCandidatesFromBlackLog();
858
859 modifyLocalContentMap(rtpContentMap);
860 final ListenableFuture<RtpContentMap> future =
861 prepareOutgoingContentMap(contentAcceptMap);
862 Futures.addCallback(
863 future,
864 new FutureCallback<>() {
865 @Override
866 public void onSuccess(final RtpContentMap rtpContentMap) {
867 sendContentAccept(rtpContentMap);
868 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
869 }
870
871 @Override
872 public void onFailure(@NonNull final Throwable throwable) {
873 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
874 }
875 },
876 MoreExecutors.directExecutor());
877 } catch (final Exception e) {
878 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
879 webRTCWrapper.close();
880 sendSessionTerminate(Reason.FAILED_APPLICATION);
881 }
882 }
883
884 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
885 final JinglePacket jinglePacket =
886 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
887 send(jinglePacket);
888 }
889
890 public synchronized void rejectContentAdd() {
891 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
892 if (incomingContentAdd == null) {
893 throw new IllegalStateException("No incoming content add");
894 }
895 this.incomingContentAdd = null;
896 updateEndUserState();
897 rejectContentAdd(incomingContentAdd);
898 }
899
900 private void rejectContentAdd(final RtpContentMap contentMap) {
901 final JinglePacket jinglePacket =
902 contentMap
903 .toStub()
904 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
905 Log.d(
906 Config.LOGTAG,
907 id.getAccount().getJid().asBareJid()
908 + ": rejecting content "
909 + ContentAddition.summary(contentMap));
910 send(jinglePacket);
911 }
912
913 private boolean checkForIceRestart(
914 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
915 final RtpContentMap existing = getRemoteContentMap();
916 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
917 final IceUdpTransportInfo.Credentials newCredentials;
918 try {
919 existingCredentials = existing.getCredentials();
920 newCredentials = rtpContentMap.getDistinctCredentials();
921 } catch (final IllegalStateException e) {
922 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
923 return false;
924 }
925 if (existingCredentials.contains(newCredentials)) {
926 return false;
927 }
928 // TODO an alternative approach is to check if we already got an iq result to our
929 // ICE-restart
930 // and if that's the case we are seeing an answer.
931 // This might be more spec compliant but also more error prone potentially
932 final boolean isSignalStateStable =
933 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
934 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
935 final boolean isOffer = rtpContentMap.emptyCandidates();
936 final RtpContentMap restartContentMap;
937 try {
938 if (isOffer) {
939 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
940 restartContentMap =
941 existing.modifiedCredentials(
942 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
943 } else {
944 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
945 Log.d(
946 Config.LOGTAG,
947 "received confirmation of ICE restart"
948 + newCredentials
949 + " peer_setup="
950 + setup);
951 // DTLS setup attribute needs to be rewritten to reflect current peer state
952 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
953 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
954 }
955 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
956 return isOffer;
957 } else {
958 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
959 respondWithTieBreak(jinglePacket);
960 return true;
961 }
962 } catch (final Exception exception) {
963 respondOk(jinglePacket);
964 final Throwable rootCause = Throwables.getRootCause(exception);
965 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
966 // If this happens a termination is already in progress
967 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
968 return true;
969 }
970 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
971 webRTCWrapper.close();
972 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
973 return true;
974 }
975 }
976
977 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
978 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
979 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
980 throw new IllegalStateException("Invalid peer setup");
981 }
982 return peerSetup;
983 }
984
985 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
986 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
987 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
988 }
989 this.peerDtlsSetup = setup;
990 }
991
992 private boolean applyIceRestart(
993 final JinglePacket jinglePacket,
994 final RtpContentMap restartContentMap,
995 final boolean isOffer)
996 throws ExecutionException, InterruptedException {
997 final SessionDescription sessionDescription =
998 SessionDescription.of(restartContentMap, isResponder());
999 final org.webrtc.SessionDescription.Type type =
1000 isOffer
1001 ? org.webrtc.SessionDescription.Type.OFFER
1002 : org.webrtc.SessionDescription.Type.ANSWER;
1003 org.webrtc.SessionDescription sdp =
1004 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1005 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1006 if (isInitiator()) {
1007 // We ignore the offer and respond with tie-break. This will clause the responder
1008 // not to apply the content map
1009 return false;
1010 }
1011 }
1012 webRTCWrapper.setRemoteDescription(sdp).get();
1013 setRemoteContentMap(restartContentMap);
1014 if (isOffer) {
1015 final SessionDescription localSessionDescription = setLocalSessionDescription();
1016 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1017 // We need to respond OK before sending any candidates
1018 respondOk(jinglePacket);
1019 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1020 } else {
1021 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1022 }
1023 return true;
1024 }
1025
1026 private void processCandidates(
1027 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1028 contents) {
1029 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1030 content : contents) {
1031 processCandidate(content);
1032 }
1033 }
1034
1035 private void processCandidate(
1036 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1037 content) {
1038 final RtpContentMap rtpContentMap = getRemoteContentMap();
1039 final List<String> indices = toIdentificationTags(rtpContentMap);
1040 final String sdpMid = content.getKey(); // aka content name
1041 final IceUdpTransportInfo transport = content.getValue().transport;
1042 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1043
1044 // TODO check that credentials remained the same
1045
1046 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1047 final String sdp;
1048 try {
1049 sdp = candidate.toSdpAttribute(credentials.ufrag);
1050 } catch (final IllegalArgumentException e) {
1051 Log.d(
1052 Config.LOGTAG,
1053 id.account.getJid().asBareJid()
1054 + ": ignoring invalid ICE candidate "
1055 + e.getMessage());
1056 continue;
1057 }
1058 final int mLineIndex = indices.indexOf(sdpMid);
1059 if (mLineIndex < 0) {
1060 Log.w(
1061 Config.LOGTAG,
1062 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1063 }
1064 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1065 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1066 this.webRTCWrapper.addIceCandidate(iceCandidate);
1067 }
1068 }
1069
1070 private RtpContentMap getRemoteContentMap() {
1071 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1072 }
1073
1074 private RtpContentMap getLocalContentMap() {
1075 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1076 }
1077
1078 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1079 final Group originalGroup = rtpContentMap.group;
1080 final List<String> identificationTags =
1081 originalGroup == null
1082 ? rtpContentMap.getNames()
1083 : originalGroup.getIdentificationTags();
1084 if (identificationTags.size() == 0) {
1085 Log.w(
1086 Config.LOGTAG,
1087 id.account.getJid().asBareJid()
1088 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1089 }
1090 return identificationTags;
1091 }
1092
1093 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1094 final JinglePacket jinglePacket, final boolean expectVerification) {
1095 try {
1096 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1097 } catch (final Exception e) {
1098 return Futures.immediateFailedFuture(e);
1099 }
1100 }
1101
1102 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1103 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1104 Log.d(
1105 Config.LOGTAG,
1106 "receiveRtpContentMap("
1107 + receivedContentMap.getClass().getSimpleName()
1108 + ",expectVerification="
1109 + expectVerification
1110 + ")");
1111 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1112 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1113 id.account
1114 .getAxolotlService()
1115 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1116 return Futures.transform(
1117 future,
1118 omemoVerifiedPayload -> {
1119 // TODO test if an exception here triggers a correct abort
1120 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1121 Log.d(
1122 Config.LOGTAG,
1123 id.account.getJid().asBareJid()
1124 + ": received verifiable DTLS fingerprint via "
1125 + omemoVerification);
1126 return omemoVerifiedPayload.getPayload();
1127 },
1128 MoreExecutors.directExecutor());
1129 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1130 return Futures.immediateFailedFuture(
1131 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1132 } else {
1133 return Futures.immediateFuture(receivedContentMap);
1134 }
1135 }
1136
1137 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1138 if (isInitiator()) {
1139 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1140 return;
1141 }
1142 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1143 Futures.addCallback(
1144 future,
1145 new FutureCallback<>() {
1146 @Override
1147 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1148 receiveSessionInitiate(jinglePacket, rtpContentMap);
1149 }
1150
1151 @Override
1152 public void onFailure(@NonNull final Throwable throwable) {
1153 respondOk(jinglePacket);
1154 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1155 }
1156 },
1157 MoreExecutors.directExecutor());
1158 }
1159
1160 private void receiveSessionInitiate(
1161 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1162 try {
1163 contentMap.requireContentDescriptions();
1164 contentMap.requireDTLSFingerprint(true);
1165 } catch (final RuntimeException e) {
1166 Log.d(
1167 Config.LOGTAG,
1168 id.account.getJid().asBareJid() + ": improperly formatted contents",
1169 Throwables.getRootCause(e));
1170 respondOk(jinglePacket);
1171 sendSessionTerminate(Reason.of(e), e.getMessage());
1172 return;
1173 }
1174 Log.d(
1175 Config.LOGTAG,
1176 "processing session-init with " + contentMap.contents.size() + " contents");
1177 final State target;
1178 if (this.state == State.PROCEED) {
1179 Preconditions.checkState(
1180 proposedMedia != null && proposedMedia.size() > 0,
1181 "proposed media must be set when processing pre-approved session-initiate");
1182 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1183 sendSessionTerminate(
1184 Reason.SECURITY_ERROR,
1185 String.format(
1186 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1187 this.proposedMedia, contentMap.getMedia()));
1188 return;
1189 }
1190 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1191 } else {
1192 target = State.SESSION_INITIALIZED;
1193 setProposedMedia(contentMap.getMedia());
1194 }
1195 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1196 respondOk(jinglePacket);
1197 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1198 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1199 Log.d(
1200 Config.LOGTAG,
1201 id.account.getJid().asBareJid()
1202 + ": automatically accepting session-initiate");
1203 sendSessionAccept();
1204 } else {
1205 Log.d(
1206 Config.LOGTAG,
1207 id.account.getJid().asBareJid()
1208 + ": received not pre-approved session-initiate. start ringing");
1209 startRinging();
1210 }
1211 } else {
1212 Log.d(
1213 Config.LOGTAG,
1214 String.format(
1215 "%s: received session-initiate while in state %s",
1216 id.account.getJid().asBareJid(), state));
1217 terminateWithOutOfOrder(jinglePacket);
1218 }
1219 }
1220
1221 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1222 if (isResponder()) {
1223 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1224 return;
1225 }
1226 final ListenableFuture<RtpContentMap> future =
1227 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1228 Futures.addCallback(
1229 future,
1230 new FutureCallback<>() {
1231 @Override
1232 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1233 receiveSessionAccept(jinglePacket, rtpContentMap);
1234 }
1235
1236 @Override
1237 public void onFailure(@NonNull final Throwable throwable) {
1238 respondOk(jinglePacket);
1239 Log.d(
1240 Config.LOGTAG,
1241 id.account.getJid().asBareJid()
1242 + ": improperly formatted contents in session-accept",
1243 throwable);
1244 webRTCWrapper.close();
1245 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1246 }
1247 },
1248 MoreExecutors.directExecutor());
1249 }
1250
1251 private void receiveSessionAccept(
1252 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1253 try {
1254 contentMap.requireContentDescriptions();
1255 contentMap.requireDTLSFingerprint();
1256 } catch (final RuntimeException e) {
1257 respondOk(jinglePacket);
1258 Log.d(
1259 Config.LOGTAG,
1260 id.account.getJid().asBareJid()
1261 + ": improperly formatted contents in session-accept",
1262 e);
1263 webRTCWrapper.close();
1264 sendSessionTerminate(Reason.of(e), e.getMessage());
1265 return;
1266 }
1267 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1268 if (!initiatorMedia.equals(contentMap.getMedia())) {
1269 sendSessionTerminate(
1270 Reason.SECURITY_ERROR,
1271 String.format(
1272 "Your session-included included media %s but our session-initiate was %s",
1273 this.proposedMedia, contentMap.getMedia()));
1274 return;
1275 }
1276 Log.d(
1277 Config.LOGTAG,
1278 "processing session-accept with " + contentMap.contents.size() + " contents");
1279 if (transition(State.SESSION_ACCEPTED)) {
1280 respondOk(jinglePacket);
1281 receiveSessionAccept(contentMap);
1282 } else {
1283 Log.d(
1284 Config.LOGTAG,
1285 String.format(
1286 "%s: received session-accept while in state %s",
1287 id.account.getJid().asBareJid(), state));
1288 respondOk(jinglePacket);
1289 }
1290 }
1291
1292 private void receiveSessionAccept(final RtpContentMap contentMap) {
1293 this.responderRtpContentMap = contentMap;
1294 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1295 final SessionDescription sessionDescription;
1296 try {
1297 sessionDescription = SessionDescription.of(contentMap, false);
1298 } catch (final IllegalArgumentException | NullPointerException e) {
1299 Log.d(
1300 Config.LOGTAG,
1301 id.account.getJid().asBareJid()
1302 + ": unable convert offer from session-accept to SDP",
1303 e);
1304 webRTCWrapper.close();
1305 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1306 return;
1307 }
1308 final org.webrtc.SessionDescription answer =
1309 new org.webrtc.SessionDescription(
1310 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1311 try {
1312 this.webRTCWrapper.setRemoteDescription(answer).get();
1313 } catch (final Exception e) {
1314 Log.d(
1315 Config.LOGTAG,
1316 id.account.getJid().asBareJid()
1317 + ": unable to set remote description after receiving session-accept",
1318 Throwables.getRootCause(e));
1319 webRTCWrapper.close();
1320 sendSessionTerminate(
1321 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1322 return;
1323 }
1324 processCandidates(contentMap.contents.entrySet());
1325 }
1326
1327 private void sendSessionAccept() {
1328 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1329 if (rtpContentMap == null) {
1330 throw new IllegalStateException("initiator RTP Content Map has not been set");
1331 }
1332 final SessionDescription offer;
1333 try {
1334 offer = SessionDescription.of(rtpContentMap, true);
1335 } catch (final IllegalArgumentException | NullPointerException e) {
1336 Log.d(
1337 Config.LOGTAG,
1338 id.account.getJid().asBareJid()
1339 + ": unable convert offer from session-initiate to SDP",
1340 e);
1341 webRTCWrapper.close();
1342 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1343 return;
1344 }
1345 sendSessionAccept(rtpContentMap.getMedia(), offer);
1346 }
1347
1348 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1349 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1350 }
1351
1352 private synchronized void sendSessionAccept(
1353 final Set<Media> media,
1354 final SessionDescription offer,
1355 final List<PeerConnection.IceServer> iceServers) {
1356 if (isTerminated()) {
1357 Log.w(
1358 Config.LOGTAG,
1359 id.account.getJid().asBareJid()
1360 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1361 return;
1362 }
1363 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1364 try {
1365 setupWebRTC(media, iceServers, !includeCandidates);
1366 } catch (final WebRTCWrapper.InitializationException e) {
1367 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1368 webRTCWrapper.close();
1369 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1370 return;
1371 }
1372 final org.webrtc.SessionDescription sdp =
1373 new org.webrtc.SessionDescription(
1374 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1375 try {
1376 this.webRTCWrapper.setRemoteDescription(sdp).get();
1377 addIceCandidatesFromBlackLog();
1378 org.webrtc.SessionDescription webRTCSessionDescription =
1379 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1380 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1381 } catch (final Exception e) {
1382 failureToAcceptSession(e);
1383 }
1384 }
1385
1386 private void failureToAcceptSession(final Throwable throwable) {
1387 if (isTerminated()) {
1388 return;
1389 }
1390 final Throwable rootCause = Throwables.getRootCause(throwable);
1391 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1392 webRTCWrapper.close();
1393 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1394 }
1395
1396 private void failureToPerformAction(
1397 final JinglePacket.Action action, final Throwable throwable) {
1398 if (isTerminated()) {
1399 return;
1400 }
1401 final Throwable rootCause = Throwables.getRootCause(throwable);
1402 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1403 webRTCWrapper.close();
1404 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1405 }
1406
1407 private void addIceCandidatesFromBlackLog() {
1408 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1409 while ((foo = this.pendingIceCandidates.poll()) != null) {
1410 processCandidate(foo);
1411 Log.d(
1412 Config.LOGTAG,
1413 id.account.getJid().asBareJid() + ": added candidate from back log");
1414 }
1415 }
1416
1417 private void prepareSessionAccept(
1418 final org.webrtc.SessionDescription webRTCSessionDescription,
1419 final boolean includeCandidates) {
1420 final SessionDescription sessionDescription =
1421 SessionDescription.parse(webRTCSessionDescription.description);
1422 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1423 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1424 if (includeCandidates) {
1425 candidates = parseCandidates(sessionDescription);
1426 } else {
1427 candidates = ImmutableMultimap.of();
1428 }
1429 this.responderRtpContentMap = respondingRtpContentMap;
1430 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1431 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1432 prepareOutgoingContentMap(respondingRtpContentMap);
1433 Futures.addCallback(
1434 outgoingContentMapFuture,
1435 new FutureCallback<>() {
1436 @Override
1437 public void onSuccess(final RtpContentMap outgoingContentMap) {
1438 if (includeCandidates) {
1439 Log.d(
1440 Config.LOGTAG,
1441 "including "
1442 + candidates.size()
1443 + " candidates in session accept");
1444 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1445 } else {
1446 sendSessionAccept(outgoingContentMap);
1447 }
1448 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1449 }
1450
1451 @Override
1452 public void onFailure(@NonNull Throwable throwable) {
1453 failureToAcceptSession(throwable);
1454 }
1455 },
1456 MoreExecutors.directExecutor());
1457 }
1458
1459 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1460 if (isTerminated()) {
1461 Log.w(
1462 Config.LOGTAG,
1463 id.account.getJid().asBareJid()
1464 + ": preparing session accept was too slow. already terminated. nothing to do.");
1465 return;
1466 }
1467 transitionOrThrow(State.SESSION_ACCEPTED);
1468 final JinglePacket sessionAccept =
1469 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1470 send(sessionAccept);
1471 }
1472
1473 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1474 final RtpContentMap rtpContentMap) {
1475 if (this.omemoVerification.hasDeviceId()) {
1476 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1477 verifiedPayloadFuture =
1478 id.account
1479 .getAxolotlService()
1480 .encrypt(
1481 rtpContentMap,
1482 id.with,
1483 omemoVerification.getDeviceId());
1484 return Futures.transform(
1485 verifiedPayloadFuture,
1486 verifiedPayload -> {
1487 omemoVerification.setOrEnsureEqual(verifiedPayload);
1488 return verifiedPayload.getPayload();
1489 },
1490 MoreExecutors.directExecutor());
1491 } else {
1492 return Futures.immediateFuture(rtpContentMap);
1493 }
1494 }
1495
1496 synchronized void deliveryMessage(
1497 final Jid from,
1498 final Element message,
1499 final String serverMessageId,
1500 final long timestamp) {
1501 Log.d(
1502 Config.LOGTAG,
1503 id.account.getJid().asBareJid()
1504 + ": delivered message to JingleRtpConnection "
1505 + message);
1506 switch (message.getName()) {
1507 case "propose" -> receivePropose(
1508 from, Propose.upgrade(message), serverMessageId, timestamp);
1509 case "proceed" -> receiveProceed(
1510 from, Proceed.upgrade(message), serverMessageId, timestamp);
1511 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1512 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1513 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1514 }
1515 }
1516
1517 void deliverFailedProceed(final String message) {
1518 Log.d(
1519 Config.LOGTAG,
1520 id.account.getJid().asBareJid()
1521 + ": receive message error for proceed message ("
1522 + Strings.nullToEmpty(message)
1523 + ")");
1524 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1525 webRTCWrapper.close();
1526 Log.d(
1527 Config.LOGTAG,
1528 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1529 this.finish();
1530 }
1531 }
1532
1533 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1534 final boolean originatedFromMyself =
1535 from.asBareJid().equals(id.account.getJid().asBareJid());
1536 if (originatedFromMyself) {
1537 if (transition(State.ACCEPTED)) {
1538 acceptedOnOtherDevice(serverMsgId, timestamp);
1539 } else {
1540 Log.d(
1541 Config.LOGTAG,
1542 id.account.getJid().asBareJid()
1543 + ": unable to transition to accept because already in state="
1544 + this.state);
1545 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1546 }
1547 } else {
1548 Log.d(
1549 Config.LOGTAG,
1550 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1551 }
1552 }
1553
1554 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1555 if (serverMsgId != null) {
1556 this.message.setServerMsgId(serverMsgId);
1557 }
1558 this.message.setTime(timestamp);
1559 this.message.setCarbon(true); // indicate that call was accepted on other device
1560 this.writeLogMessageSuccess(0);
1561 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1562 this.finish();
1563 }
1564
1565 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1566 final boolean originatedFromMyself =
1567 from.asBareJid().equals(id.account.getJid().asBareJid());
1568 // reject from another one of my clients
1569 if (originatedFromMyself) {
1570 receiveRejectFromMyself(serverMsgId, timestamp);
1571 } else if (isInitiator()) {
1572 if (from.equals(id.with)) {
1573 receiveRejectFromResponder();
1574 } else {
1575 Log.d(
1576 Config.LOGTAG,
1577 id.account.getJid()
1578 + ": ignoring reject from "
1579 + from
1580 + " for session with "
1581 + id.with);
1582 }
1583 } else {
1584 Log.d(
1585 Config.LOGTAG,
1586 id.account.getJid()
1587 + ": ignoring reject from "
1588 + from
1589 + " for session with "
1590 + id.with);
1591 }
1592 }
1593
1594 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1595 if (transition(State.REJECTED)) {
1596 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1597 this.finish();
1598 if (serverMsgId != null) {
1599 this.message.setServerMsgId(serverMsgId);
1600 }
1601 this.message.setTime(timestamp);
1602 this.message.setCarbon(true); // indicate that call was rejected on other device
1603 writeLogMessageMissed();
1604 } else {
1605 Log.d(
1606 Config.LOGTAG,
1607 "not able to transition into REJECTED because already in " + this.state);
1608 }
1609 }
1610
1611 private void receiveRejectFromResponder() {
1612 if (isInState(State.PROCEED)) {
1613 Log.d(
1614 Config.LOGTAG,
1615 id.account.getJid()
1616 + ": received reject while still in proceed. callee reconsidered");
1617 closeTransitionLogFinish(State.REJECTED_RACED);
1618 return;
1619 }
1620 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1621 Log.d(
1622 Config.LOGTAG,
1623 id.account.getJid()
1624 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1625 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1626 return;
1627 }
1628 Log.d(
1629 Config.LOGTAG,
1630 id.account.getJid()
1631 + ": ignoring reject from responder because already in state "
1632 + this.state);
1633 }
1634
1635 private void receivePropose(
1636 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1637 final boolean originatedFromMyself =
1638 from.asBareJid().equals(id.account.getJid().asBareJid());
1639 if (originatedFromMyself) {
1640 Log.d(
1641 Config.LOGTAG,
1642 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1643 } else if (transition(
1644 State.PROPOSED,
1645 () -> {
1646 final Collection<RtpDescription> descriptions =
1647 Collections2.transform(
1648 Collections2.filter(
1649 propose.getDescriptions(),
1650 d -> d instanceof RtpDescription),
1651 input -> (RtpDescription) input);
1652 final Collection<Media> media =
1653 Collections2.transform(descriptions, RtpDescription::getMedia);
1654 Preconditions.checkState(
1655 !media.contains(Media.UNKNOWN),
1656 "RTP descriptions contain unknown media");
1657 Log.d(
1658 Config.LOGTAG,
1659 id.account.getJid().asBareJid()
1660 + ": received session proposal from "
1661 + from
1662 + " for "
1663 + media);
1664 this.setProposedMedia(Sets.newHashSet(media));
1665 })) {
1666 if (serverMsgId != null) {
1667 this.message.setServerMsgId(serverMsgId);
1668 }
1669 this.message.setTime(timestamp);
1670 startRinging();
1671 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1672 sendJingleMessage("ringing");
1673 }
1674 } else {
1675 Log.d(
1676 Config.LOGTAG,
1677 id.account.getJid()
1678 + ": ignoring session proposal because already in "
1679 + state);
1680 }
1681 }
1682
1683 private void startRinging() {
1684 this.callIntegration.setRinging();
1685 Log.d(
1686 Config.LOGTAG,
1687 id.account.getJid().asBareJid()
1688 + ": received call from "
1689 + id.with
1690 + ". start ringing");
1691 ringingTimeoutFuture =
1692 jingleConnectionManager.schedule(
1693 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1694 if (CallIntegration.selfManaged()) {
1695 return;
1696 }
1697 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1698 }
1699
1700 private synchronized void ringingTimeout() {
1701 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1702 switch (this.state) {
1703 case PROPOSED -> {
1704 message.markUnread();
1705 rejectCallFromProposed();
1706 }
1707 case SESSION_INITIALIZED -> {
1708 message.markUnread();
1709 rejectCallFromSessionInitiate();
1710 }
1711 }
1712 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1713 }
1714
1715 private void cancelRingingTimeout() {
1716 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1717 if (future != null && !future.isCancelled()) {
1718 future.cancel(false);
1719 }
1720 }
1721
1722 private void receiveProceed(
1723 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1724 final Set<Media> media =
1725 Preconditions.checkNotNull(
1726 this.proposedMedia, "Proposed media has to be set before handling proceed");
1727 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1728 if (from.equals(id.with)) {
1729 if (isInitiator()) {
1730 if (transition(State.PROCEED)) {
1731 if (serverMsgId != null) {
1732 this.message.setServerMsgId(serverMsgId);
1733 }
1734 this.message.setTime(timestamp);
1735 final Integer remoteDeviceId = proceed.getDeviceId();
1736 if (isOmemoEnabled()) {
1737 this.omemoVerification.setDeviceId(remoteDeviceId);
1738 } else {
1739 if (remoteDeviceId != null) {
1740 Log.d(
1741 Config.LOGTAG,
1742 id.account.getJid().asBareJid()
1743 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1744 }
1745 this.omemoVerification.setDeviceId(null);
1746 }
1747 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1748 } else {
1749 Log.d(
1750 Config.LOGTAG,
1751 String.format(
1752 "%s: ignoring proceed because already in %s",
1753 id.account.getJid().asBareJid(), this.state));
1754 }
1755 } else {
1756 Log.d(
1757 Config.LOGTAG,
1758 String.format(
1759 "%s: ignoring proceed because we were not initializing",
1760 id.account.getJid().asBareJid()));
1761 }
1762 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1763 if (transition(State.ACCEPTED)) {
1764 Log.d(
1765 Config.LOGTAG,
1766 id.account.getJid().asBareJid()
1767 + ": moved session with "
1768 + id.with
1769 + " into state accepted after received carbon copied proceed");
1770 acceptedOnOtherDevice(serverMsgId, timestamp);
1771 }
1772 } else {
1773 Log.d(
1774 Config.LOGTAG,
1775 String.format(
1776 "%s: ignoring proceed from %s. was expected from %s",
1777 id.account.getJid().asBareJid(), from, id.with));
1778 }
1779 }
1780
1781 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1782 if (from.equals(id.with)) {
1783 final State target =
1784 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1785 if (transition(target)) {
1786 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1787 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1788 Log.d(
1789 Config.LOGTAG,
1790 id.account.getJid().asBareJid()
1791 + ": session with "
1792 + id.with
1793 + " has been retracted (serverMsgId="
1794 + serverMsgId
1795 + ")");
1796 if (serverMsgId != null) {
1797 this.message.setServerMsgId(serverMsgId);
1798 }
1799 this.message.setTime(timestamp);
1800 if (target == State.RETRACTED) {
1801 this.message.markUnread();
1802 }
1803 writeLogMessageMissed();
1804 finish();
1805 } else {
1806 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1807 }
1808 } else {
1809 // TODO parse retract from self
1810 Log.d(
1811 Config.LOGTAG,
1812 id.account.getJid().asBareJid()
1813 + ": received retract from "
1814 + from
1815 + ". expected retract from"
1816 + id.with
1817 + ". ignoring");
1818 }
1819 }
1820
1821 public void sendSessionInitiate() {
1822 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1823 }
1824
1825 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1826 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1827 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1828 }
1829
1830 private synchronized void sendSessionInitiate(
1831 final Set<Media> media,
1832 final State targetState,
1833 final List<PeerConnection.IceServer> iceServers) {
1834 if (isTerminated()) {
1835 Log.w(
1836 Config.LOGTAG,
1837 id.account.getJid().asBareJid()
1838 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1839 return;
1840 }
1841 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1842 try {
1843 setupWebRTC(media, iceServers, !includeCandidates);
1844 } catch (final WebRTCWrapper.InitializationException e) {
1845 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1846 webRTCWrapper.close();
1847 sendRetract(Reason.ofThrowable(e));
1848 return;
1849 }
1850 try {
1851 org.webrtc.SessionDescription webRTCSessionDescription =
1852 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1853 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1854 } catch (final Exception e) {
1855 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1856 // exceptions
1857 failureToInitiateSession(e, targetState);
1858 }
1859 }
1860
1861 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1862 if (isTerminated()) {
1863 return;
1864 }
1865 Log.d(
1866 Config.LOGTAG,
1867 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1868 Throwables.getRootCause(throwable));
1869 webRTCWrapper.close();
1870 final Reason reason = Reason.ofThrowable(throwable);
1871 if (isInState(targetState)) {
1872 sendSessionTerminate(reason, throwable.getMessage());
1873 } else {
1874 sendRetract(reason);
1875 }
1876 }
1877
1878 private void sendRetract(final Reason reason) {
1879 // TODO embed reason into retract
1880 sendJingleMessage("retract", id.with.asBareJid());
1881 transitionOrThrow(reasonToState(reason));
1882 this.finish();
1883 }
1884
1885 private void prepareSessionInitiate(
1886 final org.webrtc.SessionDescription webRTCSessionDescription,
1887 final boolean includeCandidates,
1888 final State targetState) {
1889 final SessionDescription sessionDescription =
1890 SessionDescription.parse(webRTCSessionDescription.description);
1891 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1892 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1893 if (includeCandidates) {
1894 candidates = parseCandidates(sessionDescription);
1895 } else {
1896 candidates = ImmutableMultimap.of();
1897 }
1898 this.initiatorRtpContentMap = rtpContentMap;
1899 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1900 encryptSessionInitiate(rtpContentMap);
1901 Futures.addCallback(
1902 outgoingContentMapFuture,
1903 new FutureCallback<>() {
1904 @Override
1905 public void onSuccess(final RtpContentMap outgoingContentMap) {
1906 if (includeCandidates) {
1907 Log.d(
1908 Config.LOGTAG,
1909 "including "
1910 + candidates.size()
1911 + " candidates in session initiate");
1912 sendSessionInitiate(
1913 outgoingContentMap.withCandidates(candidates), targetState);
1914 } else {
1915 sendSessionInitiate(outgoingContentMap, targetState);
1916 }
1917 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1918 }
1919
1920 @Override
1921 public void onFailure(@NonNull final Throwable throwable) {
1922 failureToInitiateSession(throwable, targetState);
1923 }
1924 },
1925 MoreExecutors.directExecutor());
1926 }
1927
1928 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1929 if (isTerminated()) {
1930 Log.w(
1931 Config.LOGTAG,
1932 id.account.getJid().asBareJid()
1933 + ": preparing session was too slow. already terminated. nothing to do.");
1934 return;
1935 }
1936 this.transitionOrThrow(targetState);
1937 final JinglePacket sessionInitiate =
1938 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1939 send(sessionInitiate);
1940 }
1941
1942 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1943 final RtpContentMap rtpContentMap) {
1944 if (this.omemoVerification.hasDeviceId()) {
1945 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1946 verifiedPayloadFuture =
1947 id.account
1948 .getAxolotlService()
1949 .encrypt(
1950 rtpContentMap,
1951 id.with,
1952 omemoVerification.getDeviceId());
1953 final ListenableFuture<RtpContentMap> future =
1954 Futures.transform(
1955 verifiedPayloadFuture,
1956 verifiedPayload -> {
1957 omemoVerification.setSessionFingerprint(
1958 verifiedPayload.getFingerprint());
1959 return verifiedPayload.getPayload();
1960 },
1961 MoreExecutors.directExecutor());
1962 if (Config.REQUIRE_RTP_VERIFICATION) {
1963 return future;
1964 }
1965 return Futures.catching(
1966 future,
1967 CryptoFailedException.class,
1968 e -> {
1969 Log.w(
1970 Config.LOGTAG,
1971 id.account.getJid().asBareJid()
1972 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1973 e);
1974 return rtpContentMap;
1975 },
1976 MoreExecutors.directExecutor());
1977 } else {
1978 return Futures.immediateFuture(rtpContentMap);
1979 }
1980 }
1981
1982 protected void sendSessionTerminate(final Reason reason) {
1983 sendSessionTerminate(reason, null);
1984 }
1985
1986 protected void sendSessionTerminate(final Reason reason, final String text) {
1987 sendSessionTerminate(reason, text, this::writeLogMessage);
1988 }
1989
1990 private void sendTransportInfo(
1991 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1992 final RtpContentMap transportInfo;
1993 try {
1994 final RtpContentMap rtpContentMap =
1995 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1996 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1997 } catch (final Exception e) {
1998 Log.d(
1999 Config.LOGTAG,
2000 id.account.getJid().asBareJid()
2001 + ": unable to prepare transport-info from candidate for content="
2002 + contentName);
2003 return;
2004 }
2005 final JinglePacket jinglePacket =
2006 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2007 send(jinglePacket);
2008 }
2009
2010 public RtpEndUserState getEndUserState() {
2011 switch (this.state) {
2012 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2013 if (isInitiator()) {
2014 return RtpEndUserState.RINGING;
2015 } else {
2016 return RtpEndUserState.INCOMING_CALL;
2017 }
2018 }
2019 case PROCEED -> {
2020 if (isInitiator()) {
2021 return RtpEndUserState.RINGING;
2022 } else {
2023 return RtpEndUserState.ACCEPTING_CALL;
2024 }
2025 }
2026 case SESSION_INITIALIZED_PRE_APPROVED -> {
2027 if (isInitiator()) {
2028 return RtpEndUserState.RINGING;
2029 } else {
2030 return RtpEndUserState.CONNECTING;
2031 }
2032 }
2033 case SESSION_ACCEPTED -> {
2034 final ContentAddition ca = getPendingContentAddition();
2035 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2036 return RtpEndUserState.INCOMING_CONTENT_ADD;
2037 }
2038 return getPeerConnectionStateAsEndUserState();
2039 }
2040 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2041 if (isInitiator()) {
2042 return RtpEndUserState.DECLINED_OR_BUSY;
2043 } else {
2044 return RtpEndUserState.ENDED;
2045 }
2046 }
2047 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2048 return RtpEndUserState.ENDED;
2049 }
2050 case RETRACTED_RACED -> {
2051 if (isInitiator()) {
2052 return RtpEndUserState.ENDED;
2053 } else {
2054 return RtpEndUserState.RETRACTED;
2055 }
2056 }
2057 case TERMINATED_CONNECTIVITY_ERROR -> {
2058 return zeroDuration()
2059 ? RtpEndUserState.CONNECTIVITY_ERROR
2060 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2061 }
2062 case TERMINATED_APPLICATION_FAILURE -> {
2063 return RtpEndUserState.APPLICATION_ERROR;
2064 }
2065 case TERMINATED_SECURITY_ERROR -> {
2066 return RtpEndUserState.SECURITY_ERROR;
2067 }
2068 }
2069 throw new IllegalStateException(
2070 String.format("%s has no equivalent EndUserState", this.state));
2071 }
2072
2073 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2074 final PeerConnection.PeerConnectionState state;
2075 try {
2076 state = webRTCWrapper.getState();
2077 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2078 // We usually close the WebRTCWrapper *before* transitioning so we might still
2079 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2080 return RtpEndUserState.ENDING_CALL;
2081 }
2082 return switch (state) {
2083 case CONNECTED -> RtpEndUserState.CONNECTED;
2084 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2085 case CLOSED -> RtpEndUserState.ENDING_CALL;
2086 default -> zeroDuration()
2087 ? RtpEndUserState.CONNECTIVITY_ERROR
2088 : RtpEndUserState.RECONNECTING;
2089 };
2090 }
2091
2092 private boolean isPeerConnectionConnected() {
2093 try {
2094 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2095 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2096 return false;
2097 }
2098 }
2099
2100 private void updateCallIntegrationState() {
2101 switch (this.state) {
2102 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2103 if (isInitiator()) {
2104 this.callIntegration.setDialing();
2105 } else {
2106 this.callIntegration.setRinging();
2107 }
2108 }
2109 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2110 if (isInitiator()) {
2111 this.callIntegration.setDialing();
2112 } else {
2113 this.callIntegration.setInitialized();
2114 }
2115 }
2116 case SESSION_ACCEPTED -> {
2117 if (isPeerConnectionConnected()) {
2118 this.callIntegration.setActive();
2119 } else {
2120 this.callIntegration.setInitialized();
2121 }
2122 }
2123 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2124 if (isInitiator()) {
2125 this.callIntegration.busy();
2126 } else {
2127 this.callIntegration.rejected();
2128 }
2129 }
2130 case TERMINATED_SUCCESS -> this.callIntegration.success();
2131 case ACCEPTED -> this.callIntegration.accepted();
2132 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2133 .retracted();
2134 case TERMINATED_CONNECTIVITY_ERROR,
2135 TERMINATED_APPLICATION_FAILURE,
2136 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2137 default -> throw new IllegalStateException(
2138 String.format("%s is not handled", this.state));
2139 }
2140 }
2141
2142 public ContentAddition getPendingContentAddition() {
2143 final RtpContentMap in = this.incomingContentAdd;
2144 final RtpContentMap out = this.outgoingContentAdd;
2145 if (out != null) {
2146 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2147 } else if (in != null) {
2148 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2149 } else {
2150 return null;
2151 }
2152 }
2153
2154 public Set<Media> getMedia() {
2155 final State current = getState();
2156 if (current == State.NULL) {
2157 if (isInitiator()) {
2158 return Preconditions.checkNotNull(
2159 this.proposedMedia, "RTP connection has not been initialized properly");
2160 }
2161 throw new IllegalStateException("RTP connection has not been initialized yet");
2162 }
2163 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2164 return Preconditions.checkNotNull(
2165 this.proposedMedia, "RTP connection has not been initialized properly");
2166 }
2167 final RtpContentMap localContentMap = getLocalContentMap();
2168 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2169 if (localContentMap != null) {
2170 return localContentMap.getMedia();
2171 } else if (initiatorContentMap != null) {
2172 return initiatorContentMap.getMedia();
2173 } else if (isTerminated()) {
2174 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2175 } else {
2176 return Preconditions.checkNotNull(
2177 this.proposedMedia, "RTP connection has not been initialized properly");
2178 }
2179 }
2180
2181 public boolean isVerified() {
2182 final String fingerprint = this.omemoVerification.getFingerprint();
2183 if (fingerprint == null) {
2184 return false;
2185 }
2186 final FingerprintStatus status =
2187 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2188 return status != null && status.isVerified();
2189 }
2190
2191 public boolean addMedia(final Media media) {
2192 final Set<Media> currentMedia = getMedia();
2193 if (currentMedia.contains(media)) {
2194 throw new IllegalStateException(String.format("%s has already been proposed", media));
2195 }
2196 // TODO add state protection - can only add while ACCEPTED or so
2197 Log.d(Config.LOGTAG, "adding media: " + media);
2198 return webRTCWrapper.addTrack(media);
2199 }
2200
2201 public synchronized void acceptCall() {
2202 switch (this.state) {
2203 case PROPOSED -> {
2204 cancelRingingTimeout();
2205 acceptCallFromProposed();
2206 }
2207 case SESSION_INITIALIZED -> {
2208 cancelRingingTimeout();
2209 acceptCallFromSessionInitialized();
2210 }
2211 case ACCEPTED -> Log.w(
2212 Config.LOGTAG,
2213 id.account.getJid().asBareJid()
2214 + ": the call has already been accepted with another client. UI was just lagging behind");
2215 case PROCEED, SESSION_ACCEPTED -> Log.w(
2216 Config.LOGTAG,
2217 id.account.getJid().asBareJid()
2218 + ": the call has already been accepted. user probably double tapped the UI");
2219 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2220 }
2221 }
2222
2223 public synchronized void rejectCall() {
2224 if (isTerminated()) {
2225 Log.w(
2226 Config.LOGTAG,
2227 id.account.getJid().asBareJid()
2228 + ": received rejectCall() when session has already been terminated. nothing to do");
2229 return;
2230 }
2231 switch (this.state) {
2232 case PROPOSED -> rejectCallFromProposed();
2233 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2234 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2235 }
2236 }
2237
2238 public synchronized void endCall() {
2239 if (isTerminated()) {
2240 Log.w(
2241 Config.LOGTAG,
2242 id.account.getJid().asBareJid()
2243 + ": received endCall() when session has already been terminated. nothing to do");
2244 return;
2245 }
2246 if (isInState(State.PROPOSED) && isResponder()) {
2247 rejectCallFromProposed();
2248 return;
2249 }
2250 if (isInState(State.PROCEED)) {
2251 if (isInitiator()) {
2252 retractFromProceed();
2253 } else {
2254 rejectCallFromProceed();
2255 }
2256 return;
2257 }
2258 if (isInitiator()
2259 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2260 this.webRTCWrapper.close();
2261 sendSessionTerminate(Reason.CANCEL);
2262 return;
2263 }
2264 if (isInState(State.SESSION_INITIALIZED)) {
2265 rejectCallFromSessionInitiate();
2266 return;
2267 }
2268 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2269 this.webRTCWrapper.close();
2270 sendSessionTerminate(Reason.SUCCESS);
2271 return;
2272 }
2273 if (isInState(
2274 State.TERMINATED_APPLICATION_FAILURE,
2275 State.TERMINATED_CONNECTIVITY_ERROR,
2276 State.TERMINATED_DECLINED_OR_BUSY)) {
2277 Log.d(
2278 Config.LOGTAG,
2279 "ignoring request to end call because already in state " + this.state);
2280 return;
2281 }
2282 throw new IllegalStateException(
2283 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2284 }
2285
2286 private void retractFromProceed() {
2287 Log.d(Config.LOGTAG, "retract from proceed");
2288 this.sendJingleMessage("retract");
2289 closeTransitionLogFinish(State.RETRACTED_RACED);
2290 }
2291
2292 private void closeTransitionLogFinish(final State state) {
2293 this.webRTCWrapper.close();
2294 transitionOrThrow(state);
2295 writeLogMessage(state);
2296 finish();
2297 }
2298
2299 private void setupWebRTC(
2300 final Set<Media> media,
2301 final List<PeerConnection.IceServer> iceServers,
2302 final boolean trickle)
2303 throws WebRTCWrapper.InitializationException {
2304 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2305 this.webRTCWrapper.setup(this.xmppConnectionService);
2306 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2307 }
2308
2309 private void acceptCallFromProposed() {
2310 transitionOrThrow(State.PROCEED);
2311 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2312 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2313 this.sendJingleMessage("proceed");
2314 }
2315
2316 private void rejectCallFromProposed() {
2317 transitionOrThrow(State.REJECTED);
2318 writeLogMessageMissed();
2319 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2320 this.sendJingleMessage("reject");
2321 finish();
2322 }
2323
2324 private void rejectCallFromProceed() {
2325 this.sendJingleMessage("reject");
2326 closeTransitionLogFinish(State.REJECTED_RACED);
2327 }
2328
2329 private void rejectCallFromSessionInitiate() {
2330 webRTCWrapper.close();
2331 sendSessionTerminate(Reason.DECLINE);
2332 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2333 }
2334
2335 private void sendJingleMessage(final String action) {
2336 sendJingleMessage(action, id.with);
2337 }
2338
2339 private void sendJingleMessage(final String action, final Jid to) {
2340 final MessagePacket messagePacket = new MessagePacket();
2341 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2342 messagePacket.setTo(to);
2343 final Element intent =
2344 messagePacket
2345 .addChild(action, Namespace.JINGLE_MESSAGE)
2346 .setAttribute("id", id.sessionId);
2347 if ("proceed".equals(action)) {
2348 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2349 if (isOmemoEnabled()) {
2350 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2351 final Element device =
2352 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2353 device.setAttribute("id", deviceId);
2354 }
2355 }
2356 messagePacket.addChild("store", "urn:xmpp:hints");
2357 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2358 }
2359
2360 private boolean isOmemoEnabled() {
2361 final Conversational conversational = message.getConversation();
2362 if (conversational instanceof Conversation) {
2363 return ((Conversation) conversational).getNextEncryption()
2364 == Message.ENCRYPTION_AXOLOTL;
2365 }
2366 return false;
2367 }
2368
2369 private void acceptCallFromSessionInitialized() {
2370 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2371 sendSessionAccept();
2372 }
2373
2374 @Override
2375 protected synchronized boolean transition(final State target, final Runnable runnable) {
2376 if (super.transition(target, runnable)) {
2377 updateEndUserState();
2378 updateOngoingCallNotification();
2379 return true;
2380 } else {
2381 return false;
2382 }
2383 }
2384
2385 @Override
2386 public void onIceCandidate(final IceCandidate iceCandidate) {
2387 final RtpContentMap rtpContentMap =
2388 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2389 final IceUdpTransportInfo.Credentials credentials;
2390 try {
2391 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2392 } catch (final IllegalArgumentException e) {
2393 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2394 return;
2395 }
2396 final String uFrag = credentials.ufrag;
2397 final IceUdpTransportInfo.Candidate candidate =
2398 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2399 if (candidate == null) {
2400 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2401 return;
2402 }
2403 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2404 sendTransportInfo(iceCandidate.sdpMid, candidate);
2405 }
2406
2407 @Override
2408 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2409 Log.d(
2410 Config.LOGTAG,
2411 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2412 this.stateHistory.add(newState);
2413 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2414 this.sessionDuration.start();
2415 updateOngoingCallNotification();
2416 } else if (this.sessionDuration.isRunning()) {
2417 this.sessionDuration.stop();
2418 updateOngoingCallNotification();
2419 }
2420
2421 final boolean neverConnected =
2422 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2423
2424 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2425 if (neverConnected) {
2426 if (isTerminated()) {
2427 Log.d(
2428 Config.LOGTAG,
2429 id.account.getJid().asBareJid()
2430 + ": not sending session-terminate after connectivity error because session is already in state "
2431 + this.state);
2432 return;
2433 }
2434 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2435 return;
2436 } else {
2437 this.restartIce();
2438 }
2439 }
2440 updateEndUserState();
2441 }
2442
2443 private void restartIce() {
2444 this.stateHistory.clear();
2445 this.webRTCWrapper.restartIceAsync();
2446 }
2447
2448 @Override
2449 public void onRenegotiationNeeded() {
2450 this.webRTCWrapper.execute(this::renegotiate);
2451 }
2452
2453 private void renegotiate() {
2454 final SessionDescription sessionDescription;
2455 try {
2456 sessionDescription = setLocalSessionDescription();
2457 } catch (final Exception e) {
2458 final Throwable cause = Throwables.getRootCause(e);
2459 webRTCWrapper.close();
2460 if (isTerminated()) {
2461 Log.d(
2462 Config.LOGTAG,
2463 "failed to renegotiate. session was already terminated",
2464 cause);
2465 return;
2466 }
2467 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2468 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2469 return;
2470 }
2471 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2472 final RtpContentMap currentContentMap = getLocalContentMap();
2473 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2474 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2475
2476 Log.d(
2477 Config.LOGTAG,
2478 id.getAccount().getJid().asBareJid()
2479 + ": renegotiate. iceRestart="
2480 + iceRestart
2481 + " content id diff="
2482 + diff);
2483
2484 if (diff.hasModifications() && iceRestart) {
2485 webRTCWrapper.close();
2486 sendSessionTerminate(
2487 Reason.FAILED_APPLICATION,
2488 "WebRTC unexpectedly tried to modify content and transport at once");
2489 return;
2490 }
2491
2492 if (iceRestart) {
2493 initiateIceRestart(rtpContentMap);
2494 return;
2495 } else if (diff.isEmpty()) {
2496 Log.d(
2497 Config.LOGTAG,
2498 "renegotiation. nothing to do. SignalingState="
2499 + this.webRTCWrapper.getSignalingState());
2500 }
2501
2502 if (diff.added.size() > 0) {
2503 modifyLocalContentMap(rtpContentMap);
2504 sendContentAdd(rtpContentMap, diff.added);
2505 }
2506 }
2507
2508 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2509 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2510 final JinglePacket jinglePacket =
2511 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2512 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2513 jinglePacket.setTo(id.with);
2514 xmppConnectionService.sendIqPacket(
2515 id.account,
2516 jinglePacket,
2517 (account, response) -> {
2518 if (response.getType() == IqPacket.TYPE.RESULT) {
2519 Log.d(Config.LOGTAG, "received success to our ice restart");
2520 setLocalContentMap(rtpContentMap);
2521 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2522 return;
2523 }
2524 if (response.getType() == IqPacket.TYPE.ERROR) {
2525 if (isTieBreak(response)) {
2526 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2527 return;
2528 }
2529 handleIqErrorResponse(response);
2530 }
2531 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2532 handleIqTimeoutResponse(response);
2533 }
2534 });
2535 }
2536
2537 private boolean isTieBreak(final IqPacket response) {
2538 final Element error = response.findChild("error");
2539 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2540 }
2541
2542 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2543 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2544 this.outgoingContentAdd = contentAdd;
2545 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2546 prepareOutgoingContentMap(contentAdd);
2547 Futures.addCallback(
2548 outgoingContentMapFuture,
2549 new FutureCallback<>() {
2550 @Override
2551 public void onSuccess(final RtpContentMap outgoingContentMap) {
2552 sendContentAdd(outgoingContentMap);
2553 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2554 }
2555
2556 @Override
2557 public void onFailure(@NonNull Throwable throwable) {
2558 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2559 }
2560 },
2561 MoreExecutors.directExecutor());
2562 }
2563
2564 private void sendContentAdd(final RtpContentMap contentAdd) {
2565
2566 final JinglePacket jinglePacket =
2567 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2568 jinglePacket.setTo(id.with);
2569 xmppConnectionService.sendIqPacket(
2570 id.account,
2571 jinglePacket,
2572 (connection, response) -> {
2573 if (response.getType() == IqPacket.TYPE.RESULT) {
2574 Log.d(
2575 Config.LOGTAG,
2576 id.getAccount().getJid().asBareJid()
2577 + ": received ACK to our content-add");
2578 return;
2579 }
2580 if (response.getType() == IqPacket.TYPE.ERROR) {
2581 if (isTieBreak(response)) {
2582 this.outgoingContentAdd = null;
2583 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2584 return;
2585 }
2586 handleIqErrorResponse(response);
2587 }
2588 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2589 handleIqTimeoutResponse(response);
2590 }
2591 });
2592 }
2593
2594 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2595 if (isInitiator()) {
2596 this.initiatorRtpContentMap = rtpContentMap;
2597 } else {
2598 this.responderRtpContentMap = rtpContentMap;
2599 }
2600 }
2601
2602 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2603 if (isInitiator()) {
2604 this.responderRtpContentMap = rtpContentMap;
2605 } else {
2606 this.initiatorRtpContentMap = rtpContentMap;
2607 }
2608 }
2609
2610 // this method is to be used for content map modifications that modify media
2611 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2612 final RtpContentMap activeContents = rtpContentMap.activeContents();
2613 setLocalContentMap(activeContents);
2614 this.callIntegration.setAudioDeviceWhenAvailable(
2615 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2616 updateEndUserState();
2617 }
2618
2619 private SessionDescription setLocalSessionDescription()
2620 throws ExecutionException, InterruptedException {
2621 final org.webrtc.SessionDescription sessionDescription =
2622 this.webRTCWrapper.setLocalDescription(false).get();
2623 return SessionDescription.parse(sessionDescription.description);
2624 }
2625
2626 private void closeWebRTCSessionAfterFailedConnection() {
2627 this.webRTCWrapper.close();
2628 synchronized (this) {
2629 if (isTerminated()) {
2630 Log.d(
2631 Config.LOGTAG,
2632 id.account.getJid().asBareJid()
2633 + ": no need to send session-terminate after failed connection. Other party already did");
2634 return;
2635 }
2636 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2637 }
2638 }
2639
2640 public boolean zeroDuration() {
2641 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2642 }
2643
2644 public long getCallDuration() {
2645 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2646 }
2647
2648 public CallIntegration getCallIntegration() {
2649 return this.callIntegration;
2650 }
2651
2652 public boolean isMicrophoneEnabled() {
2653 return webRTCWrapper.isMicrophoneEnabled();
2654 }
2655
2656 public boolean setMicrophoneEnabled(final boolean enabled) {
2657 return webRTCWrapper.setMicrophoneEnabled(enabled);
2658 }
2659
2660 public boolean isVideoEnabled() {
2661 return webRTCWrapper.isVideoEnabled();
2662 }
2663
2664 public void setVideoEnabled(final boolean enabled) {
2665 webRTCWrapper.setVideoEnabled(enabled);
2666 }
2667
2668 public boolean isCameraSwitchable() {
2669 return webRTCWrapper.isCameraSwitchable();
2670 }
2671
2672 public boolean isFrontCamera() {
2673 return webRTCWrapper.isFrontCamera();
2674 }
2675
2676 public ListenableFuture<Boolean> switchCamera() {
2677 return webRTCWrapper.switchCamera();
2678 }
2679
2680 @Override
2681 public void onCallIntegrationShowIncomingCallUi() {
2682 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2683 }
2684
2685 @Override
2686 public void onCallIntegrationDisconnect() {
2687 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2688 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2689 rejectCall();
2690 } else {
2691 endCall();
2692 }
2693 }
2694
2695 @Override
2696 public void onCallIntegrationReject() {
2697 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2698 try {
2699 rejectCall();
2700 } catch (final IllegalStateException e) {
2701 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2702 }
2703 }
2704
2705 @Override
2706 public void onCallIntegrationAnswer() {
2707 // we need to start the UI to a) show it and b) be able to ask for permissions
2708 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2709 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2710 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2711 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2712 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2713 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2714 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2715 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2716 xmppConnectionService.startActivity(intent);
2717 }
2718
2719 @Override
2720 public void onAudioDeviceChanged(
2721 final CallIntegration.AudioDevice selectedAudioDevice,
2722 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2723 Log.d(
2724 Config.LOGTAG,
2725 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2726 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2727 selectedAudioDevice, availableAudioDevices);
2728 }
2729
2730 private void updateEndUserState() {
2731 final RtpEndUserState endUserState = getEndUserState();
2732 this.updateCallIntegrationState();
2733 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2734 id.account, id.with, id.sessionId, endUserState);
2735 }
2736
2737 private void updateOngoingCallNotification() {
2738 final State state = this.state;
2739 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2740 final boolean reconnecting;
2741 if (state == State.SESSION_ACCEPTED) {
2742 reconnecting =
2743 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2744 } else {
2745 reconnecting = false;
2746 }
2747 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2748 } else {
2749 xmppConnectionService.removeOngoingCall();
2750 }
2751 }
2752
2753 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2754 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2755 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2756 request.setTo(id.account.getDomain());
2757 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2758 xmppConnectionService.sendIqPacket(
2759 id.account,
2760 request,
2761 (account, response) -> {
2762 final var iceServers = IceServers.parse(response);
2763 if (iceServers.size() == 0) {
2764 Log.w(
2765 Config.LOGTAG,
2766 id.account.getJid().asBareJid()
2767 + ": no ICE server found "
2768 + response);
2769 }
2770 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2771 });
2772 } else {
2773 Log.w(
2774 Config.LOGTAG,
2775 id.account.getJid().asBareJid() + ": has no external service discovery");
2776 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2777 }
2778 }
2779
2780 @Override
2781 protected void terminateTransport() {
2782 this.webRTCWrapper.close();
2783 }
2784
2785 @Override
2786 protected void finish() {
2787 if (isTerminated()) {
2788 this.cancelRingingTimeout();
2789 this.callIntegration.verifyDisconnected();
2790 this.webRTCWrapper.verifyClosed();
2791 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2792 super.finish();
2793 } else {
2794 throw new IllegalStateException(
2795 String.format("Unable to call finish from %s", this.state));
2796 }
2797 }
2798
2799 private void writeLogMessage(final State state) {
2800 final long duration = getCallDuration();
2801 if (state == State.TERMINATED_SUCCESS
2802 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2803 writeLogMessageSuccess(duration);
2804 } else {
2805 writeLogMessageMissed();
2806 }
2807 }
2808
2809 private void writeLogMessageSuccess(final long duration) {
2810 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2811 this.writeMessage();
2812 }
2813
2814 private void writeLogMessageMissed() {
2815 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2816 this.writeMessage();
2817 }
2818
2819 private void writeMessage() {
2820 final Conversational conversational = message.getConversation();
2821 if (conversational instanceof Conversation) {
2822 ((Conversation) conversational).add(this.message);
2823 xmppConnectionService.createMessageAsync(message);
2824 xmppConnectionService.updateConversationUi();
2825 } else {
2826 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2827 }
2828 }
2829
2830 public Optional<VideoTrack> getLocalVideoTrack() {
2831 return webRTCWrapper.getLocalVideoTrack();
2832 }
2833
2834 public Optional<VideoTrack> getRemoteVideoTrack() {
2835 return webRTCWrapper.getRemoteVideoTrack();
2836 }
2837
2838 public EglBase.Context getEglBaseContext() {
2839 return webRTCWrapper.getEglBaseContext();
2840 }
2841
2842 void setProposedMedia(final Set<Media> media) {
2843 this.proposedMedia = media;
2844 this.callIntegration.setVideoState(
2845 Media.audioOnly(media)
2846 ? VideoProfile.STATE_AUDIO_ONLY
2847 : VideoProfile.STATE_BIDIRECTIONAL);
2848 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2849 }
2850
2851 public void fireStateUpdate() {
2852 final RtpEndUserState endUserState = getEndUserState();
2853 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2854 id.account, id.with, id.sessionId, endUserState);
2855 }
2856
2857 public boolean isSwitchToVideoAvailable() {
2858 final boolean prerequisite =
2859 Media.audioOnly(getMedia())
2860 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2861 .contains(getEndUserState());
2862 return prerequisite && remoteHasVideoFeature();
2863 }
2864
2865 private boolean remoteHasVideoFeature() {
2866 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2867 }
2868
2869 private boolean remoteHasSdpOfferAnswer() {
2870 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2871 }
2872
2873 private interface OnIceServersDiscovered {
2874 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2875 }
2876}