1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMultimap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Iterables;
19import com.google.common.collect.Maps;
20import com.google.common.collect.Sets;
21import com.google.common.primitives.Ints;
22import com.google.common.util.concurrent.FutureCallback;
23import com.google.common.util.concurrent.Futures;
24import com.google.common.util.concurrent.ListenableFuture;
25import com.google.common.util.concurrent.MoreExecutors;
26
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Conversation;
33import eu.siacs.conversations.entities.Conversational;
34import eu.siacs.conversations.entities.Message;
35import eu.siacs.conversations.entities.RtpSessionStatus;
36import eu.siacs.conversations.services.AppRTCAudioManager;
37import eu.siacs.conversations.utils.IP;
38import eu.siacs.conversations.xml.Element;
39import eu.siacs.conversations.xml.Namespace;
40import eu.siacs.conversations.xmpp.Jid;
41import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
42import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
43import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
44import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
48import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
49import eu.siacs.conversations.xmpp.stanzas.IqPacket;
50import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
51
52import org.webrtc.EglBase;
53import org.webrtc.IceCandidate;
54import org.webrtc.PeerConnection;
55import org.webrtc.VideoTrack;
56
57import java.util.Arrays;
58import java.util.Collection;
59import java.util.Collections;
60import java.util.LinkedList;
61import java.util.List;
62import java.util.Map;
63import java.util.Queue;
64import java.util.Set;
65import java.util.concurrent.ExecutionException;
66import java.util.concurrent.ScheduledFuture;
67import java.util.concurrent.TimeUnit;
68
69public class JingleRtpConnection extends AbstractJingleConnection
70 implements WebRTCWrapper.EventCallback {
71
72 public static final List<State> STATES_SHOWING_ONGOING_CALL =
73 Arrays.asList(
74 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
75 private static final long BUSY_TIME_OUT = 30;
76
77 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
78 private final Queue<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>>
79 pendingIceCandidates = new LinkedList<>();
80 private final OmemoVerification omemoVerification = new OmemoVerification();
81 private final Message message;
82
83 private Set<Media> proposedMedia;
84 private RtpContentMap initiatorRtpContentMap;
85 private RtpContentMap responderRtpContentMap;
86 private RtpContentMap incomingContentAdd;
87 private RtpContentMap outgoingContentAdd;
88 private IceUdpTransportInfo.Setup peerDtlsSetup;
89 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
90 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
91 private ScheduledFuture<?> ringingTimeoutFuture;
92
93 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
94 super(jingleConnectionManager, id, initiator);
95 final Conversation conversation =
96 jingleConnectionManager
97 .getXmppConnectionService()
98 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
99 this.message =
100 new Message(
101 conversation,
102 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
103 Message.TYPE_RTP_SESSION,
104 id.sessionId);
105 }
106
107 @Override
108 synchronized void deliverPacket(final JinglePacket jinglePacket) {
109 switch (jinglePacket.getAction()) {
110 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
111 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
112 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
113 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
114 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
115 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
116 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
117 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
118 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
119 default -> {
120 respondOk(jinglePacket);
121 Log.d(
122 Config.LOGTAG,
123 String.format(
124 "%s: received unhandled jingle action %s",
125 id.account.getJid().asBareJid(), jinglePacket.getAction()));
126 }
127 }
128 }
129
130 @Override
131 synchronized void notifyRebound() {
132 if (isTerminated()) {
133 return;
134 }
135 webRTCWrapper.close();
136 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
137 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
138 }
139 if (isInState(
140 State.SESSION_INITIALIZED,
141 State.SESSION_INITIALIZED_PRE_APPROVED,
142 State.SESSION_ACCEPTED)) {
143 // we might have already changed resources (full jid) at this point; so this might not
144 // even reach the other party
145 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
146 } else {
147 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
148 finish();
149 }
150 }
151
152 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
153 respondOk(jinglePacket);
154 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
155 final State previous = this.state;
156 Log.d(
157 Config.LOGTAG,
158 id.account.getJid().asBareJid()
159 + ": received session terminate reason="
160 + wrapper.reason
161 + "("
162 + Strings.nullToEmpty(wrapper.text)
163 + ") while in state "
164 + previous);
165 if (TERMINATED.contains(previous)) {
166 Log.d(
167 Config.LOGTAG,
168 id.account.getJid().asBareJid()
169 + ": ignoring session terminate because already in "
170 + previous);
171 return;
172 }
173 webRTCWrapper.close();
174 final State target = reasonToState(wrapper.reason);
175 transitionOrThrow(target);
176 writeLogMessage(target);
177 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
178 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
179 }
180 finish();
181 }
182
183 private void receiveTransportInfo(final JinglePacket jinglePacket) {
184 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
185 // INITIALIZED only after transport-info has been received
186 if (isInState(
187 State.NULL,
188 State.PROCEED,
189 State.SESSION_INITIALIZED,
190 State.SESSION_INITIALIZED_PRE_APPROVED,
191 State.SESSION_ACCEPTED)) {
192 final RtpContentMap contentMap;
193 try {
194 contentMap = RtpContentMap.of(jinglePacket);
195 } catch (final IllegalArgumentException | NullPointerException e) {
196 Log.d(
197 Config.LOGTAG,
198 id.account.getJid().asBareJid()
199 + ": improperly formatted contents; ignoring",
200 e);
201 respondOk(jinglePacket);
202 return;
203 }
204 receiveTransportInfo(jinglePacket, contentMap);
205 } else {
206 if (isTerminated()) {
207 respondOk(jinglePacket);
208 Log.d(
209 Config.LOGTAG,
210 id.account.getJid().asBareJid()
211 + ": ignoring out-of-order transport info; we where already terminated");
212 } else {
213 Log.d(
214 Config.LOGTAG,
215 id.account.getJid().asBareJid()
216 + ": received transport info while in state="
217 + this.state);
218 terminateWithOutOfOrder(jinglePacket);
219 }
220 }
221 }
222
223 private void receiveTransportInfo(
224 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
225 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> candidates =
226 contentMap.contents.entrySet();
227 final RtpContentMap remote = getRemoteContentMap();
228 final Set<String> remoteContentIds =
229 remote == null ? Collections.emptySet() : remote.contents.keySet();
230 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
231 Log.d(
232 Config.LOGTAG,
233 "received transport-info for unknown contents "
234 + contentMap.contents.keySet()
235 + " (known: "
236 + remoteContentIds
237 + ")");
238 respondOk(jinglePacket);
239 pendingIceCandidates.addAll(candidates);
240 return;
241 }
242 if (this.state != State.SESSION_ACCEPTED) {
243 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
244 respondOk(jinglePacket);
245 pendingIceCandidates.addAll(candidates);
246 return;
247 }
248 // zero candidates + modified credentials are an ICE restart offer
249 if (checkForIceRestart(jinglePacket, contentMap)) {
250 return;
251 }
252 respondOk(jinglePacket);
253 try {
254 processCandidates(candidates);
255 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
256 Log.w(
257 Config.LOGTAG,
258 id.account.getJid().asBareJid()
259 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
260 }
261 }
262
263 private void receiveContentAdd(final JinglePacket jinglePacket) {
264 final RtpContentMap modification;
265 try {
266 modification = RtpContentMap.of(jinglePacket);
267 modification.requireContentDescriptions();
268 } catch (final RuntimeException e) {
269 Log.d(
270 Config.LOGTAG,
271 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
272 Throwables.getRootCause(e));
273 respondOk(jinglePacket);
274 webRTCWrapper.close();
275 sendSessionTerminate(Reason.of(e), e.getMessage());
276 return;
277 }
278 if (isInState(State.SESSION_ACCEPTED)) {
279 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
280 final ListenableFuture<RtpContentMap> future =
281 receiveRtpContentMap(
282 modification,
283 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
284 Futures.addCallback(
285 future,
286 new FutureCallback<>() {
287 @Override
288 public void onSuccess(final RtpContentMap rtpContentMap) {
289 receiveContentAdd(jinglePacket, rtpContentMap);
290 }
291
292 @Override
293 public void onFailure(@NonNull Throwable throwable) {
294 respondOk(jinglePacket);
295 final Throwable rootCause = Throwables.getRootCause(throwable);
296 Log.d(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": improperly formatted contents in content-add",
300 throwable);
301 webRTCWrapper.close();
302 sendSessionTerminate(
303 Reason.ofThrowable(rootCause), rootCause.getMessage());
304 }
305 },
306 MoreExecutors.directExecutor());
307 } else {
308 terminateWithOutOfOrder(jinglePacket);
309 }
310 }
311
312 private void receiveContentAdd(
313 final JinglePacket jinglePacket, final RtpContentMap modification) {
314 final RtpContentMap remote = getRemoteContentMap();
315 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
316 respondOk(jinglePacket);
317 this.webRTCWrapper.close();
318 sendSessionTerminate(
319 Reason.FAILED_APPLICATION,
320 String.format(
321 "contents with names %s already exists",
322 Joiner.on(", ").join(modification.getNames())));
323 return;
324 }
325 final ContentAddition contentAddition =
326 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
327
328 final RtpContentMap outgoing = this.outgoingContentAdd;
329 final Set<ContentAddition.Summary> outgoingContentAddSummary =
330 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
331
332 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
333 if (isInitiator()) {
334 Log.d(
335 Config.LOGTAG,
336 id.getAccount().getJid().asBareJid()
337 + ": respond with tie break to matching content-add offer");
338 respondWithTieBreak(jinglePacket);
339 } else {
340 Log.d(
341 Config.LOGTAG,
342 id.getAccount().getJid().asBareJid()
343 + ": automatically accept matching content-add offer");
344 acceptContentAdd(contentAddition.summary, modification);
345 }
346 return;
347 }
348
349 // once we can display multiple video tracks we can be more loose with this condition
350 // theoretically it should also be fine to automatically accept audio only contents
351 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
352 Log.d(
353 Config.LOGTAG,
354 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
355 this.incomingContentAdd = modification;
356 respondOk(jinglePacket);
357 updateEndUserState();
358 } else {
359 respondOk(jinglePacket);
360 // TODO do we want to add a reason?
361 rejectContentAdd(modification);
362 }
363 }
364
365 private void receiveContentAccept(final JinglePacket jinglePacket) {
366 final RtpContentMap receivedContentAccept;
367 try {
368 receivedContentAccept = RtpContentMap.of(jinglePacket);
369 receivedContentAccept.requireContentDescriptions();
370 } catch (final RuntimeException e) {
371 Log.d(
372 Config.LOGTAG,
373 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
374 Throwables.getRootCause(e));
375 respondOk(jinglePacket);
376 webRTCWrapper.close();
377 sendSessionTerminate(Reason.of(e), e.getMessage());
378 return;
379 }
380
381 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
382 if (outgoingContentAdd == null) {
383 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
384 terminateWithOutOfOrder(jinglePacket);
385 return;
386 }
387 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
388 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
389 this.outgoingContentAdd = null;
390 respondOk(jinglePacket);
391 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
392 final ListenableFuture<RtpContentMap> future =
393 receiveRtpContentMap(
394 receivedContentAccept,
395 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
396 Futures.addCallback(
397 future,
398 new FutureCallback<>() {
399 @Override
400 public void onSuccess(final RtpContentMap result) {
401 receiveContentAccept(result);
402 }
403
404 @Override
405 public void onFailure(@NonNull final Throwable throwable) {
406 webRTCWrapper.close();
407 sendSessionTerminate(
408 Reason.ofThrowable(throwable), throwable.getMessage());
409 }
410 },
411 MoreExecutors.directExecutor());
412 } else {
413 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
414 terminateWithOutOfOrder(jinglePacket);
415 }
416 }
417
418 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
419 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
420 final RtpContentMap modifiedContentMap =
421 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
422
423 setRemoteContentMap(modifiedContentMap);
424
425 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
426
427 final org.webrtc.SessionDescription sdp =
428 new org.webrtc.SessionDescription(
429 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
430
431 try {
432 this.webRTCWrapper.setRemoteDescription(sdp).get();
433 } catch (final Exception e) {
434 final Throwable cause = Throwables.getRootCause(e);
435 Log.d(
436 Config.LOGTAG,
437 id.getAccount().getJid().asBareJid()
438 + ": unable to set remote description after receiving content-accept",
439 cause);
440 webRTCWrapper.close();
441 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
442 return;
443 }
444 Log.d(
445 Config.LOGTAG,
446 id.getAccount().getJid().asBareJid()
447 + ": remote has accepted content-add "
448 + ContentAddition.summary(receivedContentAccept));
449 processCandidates(receivedContentAccept.contents.entrySet());
450 updateEndUserState();
451 }
452
453 private void receiveContentModify(final JinglePacket jinglePacket) {
454 if (this.state != State.SESSION_ACCEPTED) {
455 terminateWithOutOfOrder(jinglePacket);
456 return;
457 }
458 final Map<String, Content.Senders> modification =
459 Maps.transformEntries(
460 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
461 final boolean isInitiator = isInitiator();
462 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
463 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
464 final Set<String> currentOutgoingMediaIds =
465 currentOutgoing == null
466 ? Collections.emptySet()
467 : currentOutgoing.contents.keySet();
468 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
469 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
470 respondOk(jinglePacket);
471 final RtpContentMap modifiedContentMap;
472 try {
473 modifiedContentMap =
474 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
475 } catch (final IllegalArgumentException e) {
476 webRTCWrapper.close();
477 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
478 return;
479 }
480 this.outgoingContentAdd = modifiedContentMap;
481 Log.d(
482 Config.LOGTAG,
483 id.account.getJid().asBareJid()
484 + ": processed content-modification for pending content-add");
485 } else if (remoteContentMap != null
486 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
487 respondOk(jinglePacket);
488 final RtpContentMap modifiedRemoteContentMap;
489 try {
490 modifiedRemoteContentMap =
491 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
492 } catch (final IllegalArgumentException e) {
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
495 return;
496 }
497 final SessionDescription offer;
498 try {
499 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
500 } catch (final IllegalArgumentException | NullPointerException e) {
501 Log.d(
502 Config.LOGTAG,
503 id.getAccount().getJid().asBareJid()
504 + ": unable convert offer from content-modify to SDP",
505 e);
506 webRTCWrapper.close();
507 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
508 return;
509 }
510 Log.d(
511 Config.LOGTAG,
512 id.account.getJid().asBareJid() + ": auto accepting content-modification");
513 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
514 } else {
515 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
516 respondWithItemNotFound(jinglePacket);
517 }
518 }
519
520 private void autoAcceptContentModify(
521 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
522 this.setRemoteContentMap(modifiedRemoteContentMap);
523 final org.webrtc.SessionDescription sdp =
524 new org.webrtc.SessionDescription(
525 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
526 try {
527 this.webRTCWrapper.setRemoteDescription(sdp).get();
528 // auto accept is only done when we already have tracks
529 final SessionDescription answer = setLocalSessionDescription();
530 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
531 modifyLocalContentMap(rtpContentMap);
532 // we do not need to send an answer but do we have to resend the candidates currently in
533 // SDP?
534 // resendCandidatesFromSdp(answer);
535 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
536 } catch (final Exception e) {
537 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
538 webRTCWrapper.close();
539 sendSessionTerminate(Reason.FAILED_APPLICATION);
540 }
541 }
542
543 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
544 final SessionDescription answer) {
545 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
546 new ImmutableMultimap.Builder<>();
547 for (final SessionDescription.Media media : answer.media) {
548 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
549 if (Strings.isNullOrEmpty(mid)) {
550 continue;
551 }
552 for (final String sdpCandidate : media.attributes.get("candidate")) {
553 final IceUdpTransportInfo.Candidate candidate =
554 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
555 if (candidate != null) {
556 candidateBuilder.put(mid, candidate);
557 }
558 }
559 }
560 return candidateBuilder.build();
561 }
562
563 private void receiveContentReject(final JinglePacket jinglePacket) {
564 final RtpContentMap receivedContentReject;
565 try {
566 receivedContentReject = RtpContentMap.of(jinglePacket);
567 } catch (final RuntimeException e) {
568 Log.d(
569 Config.LOGTAG,
570 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
571 Throwables.getRootCause(e));
572 respondOk(jinglePacket);
573 this.webRTCWrapper.close();
574 sendSessionTerminate(Reason.of(e), e.getMessage());
575 return;
576 }
577
578 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
579 if (outgoingContentAdd == null) {
580 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
581 terminateWithOutOfOrder(jinglePacket);
582 return;
583 }
584 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
585 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
586 this.outgoingContentAdd = null;
587 respondOk(jinglePacket);
588 Log.d(Config.LOGTAG, jinglePacket.toString());
589 receiveContentReject(ourSummary);
590 } else {
591 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
592 terminateWithOutOfOrder(jinglePacket);
593 }
594 }
595
596 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
597 try {
598 this.webRTCWrapper.removeTrack(Media.VIDEO);
599 final RtpContentMap localContentMap = customRollback();
600 modifyLocalContentMap(localContentMap);
601 } catch (final Exception e) {
602 final Throwable cause = Throwables.getRootCause(e);
603 Log.d(
604 Config.LOGTAG,
605 id.getAccount().getJid().asBareJid()
606 + ": unable to rollback local description after receiving content-reject",
607 cause);
608 webRTCWrapper.close();
609 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
610 return;
611 }
612 Log.d(
613 Config.LOGTAG,
614 id.getAccount().getJid().asBareJid()
615 + ": remote has rejected our content-add "
616 + summary);
617 }
618
619 private void receiveContentRemove(final JinglePacket jinglePacket) {
620 final RtpContentMap receivedContentRemove;
621 try {
622 receivedContentRemove = RtpContentMap.of(jinglePacket);
623 receivedContentRemove.requireContentDescriptions();
624 } catch (final RuntimeException e) {
625 Log.d(
626 Config.LOGTAG,
627 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
628 Throwables.getRootCause(e));
629 respondOk(jinglePacket);
630 this.webRTCWrapper.close();
631 sendSessionTerminate(Reason.of(e), e.getMessage());
632 return;
633 }
634 respondOk(jinglePacket);
635 receiveContentRemove(receivedContentRemove);
636 }
637
638 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
639 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
640 final Set<ContentAddition.Summary> contentAddSummary =
641 incomingContentAdd == null
642 ? Collections.emptySet()
643 : ContentAddition.summary(incomingContentAdd);
644 final Set<ContentAddition.Summary> removeSummary =
645 ContentAddition.summary(receivedContentRemove);
646 if (contentAddSummary.equals(removeSummary)) {
647 this.incomingContentAdd = null;
648 updateEndUserState();
649 } else {
650 webRTCWrapper.close();
651 sendSessionTerminate(
652 Reason.FAILED_APPLICATION,
653 String.format(
654 "%s only supports %s as a means to retract a not yet accepted %s",
655 BuildConfig.APP_NAME,
656 JinglePacket.Action.CONTENT_REMOVE,
657 JinglePacket.Action.CONTENT_ADD));
658 }
659 }
660
661 public synchronized void retractContentAdd() {
662 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
663 if (outgoingContentAdd == null) {
664 throw new IllegalStateException("Not outgoing content add");
665 }
666 try {
667 webRTCWrapper.removeTrack(Media.VIDEO);
668 final RtpContentMap localContentMap = customRollback();
669 modifyLocalContentMap(localContentMap);
670 } catch (final Exception e) {
671 final Throwable cause = Throwables.getRootCause(e);
672 Log.d(
673 Config.LOGTAG,
674 id.getAccount().getJid().asBareJid()
675 + ": unable to rollback local description after trying to retract content-add",
676 cause);
677 webRTCWrapper.close();
678 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
679 return;
680 }
681 this.outgoingContentAdd = null;
682 final JinglePacket retract =
683 outgoingContentAdd
684 .toStub()
685 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
686 this.send(retract);
687 Log.d(
688 Config.LOGTAG,
689 id.getAccount().getJid()
690 + ": retract content-add "
691 + ContentAddition.summary(outgoingContentAdd));
692 }
693
694 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
695 final SessionDescription sdp = setLocalSessionDescription();
696 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
697 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
698 this.webRTCWrapper
699 .setRemoteDescription(
700 new org.webrtc.SessionDescription(
701 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
702 .get();
703 return localRtpContentMap;
704 }
705
706 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
707 final RtpContentMap currentRemote = getRemoteContentMap();
708 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
709 if (diff.isEmpty()) {
710 throw new IllegalStateException(
711 "Unexpected rollback condition. No difference between local and remote");
712 }
713 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
714 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
715 final RtpContentMap nextRemote =
716 currentRemote.addContent(
717 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
718 return SessionDescription.of(nextRemote, isResponder());
719 }
720 throw new IllegalStateException(
721 "Unexpected rollback condition. Senders were not uniformly none");
722 }
723
724 public synchronized void acceptContentAdd(
725 @NonNull final Set<ContentAddition.Summary> contentAddition) {
726 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
727 if (incomingContentAdd == null) {
728 throw new IllegalStateException("No incoming content add");
729 }
730
731 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
732 this.incomingContentAdd = null;
733 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
734 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
735 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
736 Log.d(
737 Config.LOGTAG,
738 "content addition is receive only. we want to upgrade to 'both'");
739 final RtpContentMap modifiedSenders =
740 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
741 final JinglePacket proposedContentModification =
742 modifiedSenders
743 .toStub()
744 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
745 proposedContentModification.setTo(id.with);
746 xmppConnectionService.sendIqPacket(
747 id.account,
748 proposedContentModification,
749 (account, response) -> {
750 if (response.getType() == IqPacket.TYPE.RESULT) {
751 Log.d(
752 Config.LOGTAG,
753 id.account.getJid().asBareJid()
754 + ": remote has accepted our upgrade to senders=both");
755 acceptContentAdd(
756 ContentAddition.summary(modifiedSenders), modifiedSenders);
757 } else {
758 Log.d(
759 Config.LOGTAG,
760 id.account.getJid().asBareJid()
761 + ": remote has rejected our upgrade to senders=both");
762 acceptContentAdd(contentAddition, incomingContentAdd);
763 }
764 });
765 } else {
766 acceptContentAdd(contentAddition, incomingContentAdd);
767 }
768 } else {
769 throw new IllegalStateException(
770 "Accepted content add does not match pending content-add");
771 }
772 }
773
774 private void acceptContentAdd(
775 @NonNull final Set<ContentAddition.Summary> contentAddition,
776 final RtpContentMap incomingContentAdd) {
777 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
778 final RtpContentMap modifiedContentMap =
779 getRemoteContentMap().addContent(incomingContentAdd, setup);
780 this.setRemoteContentMap(modifiedContentMap);
781
782 final SessionDescription offer;
783 try {
784 offer = SessionDescription.of(modifiedContentMap, isResponder());
785 } catch (final IllegalArgumentException | NullPointerException e) {
786 Log.d(
787 Config.LOGTAG,
788 id.getAccount().getJid().asBareJid()
789 + ": unable convert offer from content-add to SDP",
790 e);
791 webRTCWrapper.close();
792 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
793 return;
794 }
795 this.incomingContentAdd = null;
796 acceptContentAdd(contentAddition, offer);
797 }
798
799 private void acceptContentAdd(
800 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
801 final org.webrtc.SessionDescription sdp =
802 new org.webrtc.SessionDescription(
803 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
804 try {
805 this.webRTCWrapper.setRemoteDescription(sdp).get();
806
807 // TODO add tracks for 'media' where contentAddition.senders matches
808
809 // TODO if senders.sending(isInitiator())
810
811 this.webRTCWrapper.addTrack(Media.VIDEO);
812
813 // TODO add additional transceivers for recv only cases
814
815 final SessionDescription answer = setLocalSessionDescription();
816 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
817
818 final RtpContentMap contentAcceptMap =
819 rtpContentMap.toContentModification(
820 Collections2.transform(contentAddition, ca -> ca.name));
821
822 Log.d(
823 Config.LOGTAG,
824 id.getAccount().getJid().asBareJid()
825 + ": sending content-accept "
826 + ContentAddition.summary(contentAcceptMap));
827
828 addIceCandidatesFromBlackLog();
829
830 modifyLocalContentMap(rtpContentMap);
831 final ListenableFuture<RtpContentMap> future =
832 prepareOutgoingContentMap(contentAcceptMap);
833 Futures.addCallback(
834 future,
835 new FutureCallback<>() {
836 @Override
837 public void onSuccess(final RtpContentMap rtpContentMap) {
838 sendContentAccept(rtpContentMap);
839 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
840 }
841
842 @Override
843 public void onFailure(@NonNull final Throwable throwable) {
844 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
845 }
846 },
847 MoreExecutors.directExecutor());
848 } catch (final Exception e) {
849 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
850 webRTCWrapper.close();
851 sendSessionTerminate(Reason.FAILED_APPLICATION);
852 }
853 }
854
855 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
856 final JinglePacket jinglePacket =
857 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
858 send(jinglePacket);
859 }
860
861 public synchronized void rejectContentAdd() {
862 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
863 if (incomingContentAdd == null) {
864 throw new IllegalStateException("No incoming content add");
865 }
866 this.incomingContentAdd = null;
867 updateEndUserState();
868 rejectContentAdd(incomingContentAdd);
869 }
870
871 private void rejectContentAdd(final RtpContentMap contentMap) {
872 final JinglePacket jinglePacket =
873 contentMap
874 .toStub()
875 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
876 Log.d(
877 Config.LOGTAG,
878 id.getAccount().getJid().asBareJid()
879 + ": rejecting content "
880 + ContentAddition.summary(contentMap));
881 send(jinglePacket);
882 }
883
884 private boolean checkForIceRestart(
885 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
886 final RtpContentMap existing = getRemoteContentMap();
887 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
888 final IceUdpTransportInfo.Credentials newCredentials;
889 try {
890 existingCredentials = existing.getCredentials();
891 newCredentials = rtpContentMap.getDistinctCredentials();
892 } catch (final IllegalStateException e) {
893 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
894 return false;
895 }
896 if (existingCredentials.contains(newCredentials)) {
897 return false;
898 }
899 // TODO an alternative approach is to check if we already got an iq result to our
900 // ICE-restart
901 // and if that's the case we are seeing an answer.
902 // This might be more spec compliant but also more error prone potentially
903 final boolean isSignalStateStable =
904 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
905 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
906 final boolean isOffer = rtpContentMap.emptyCandidates();
907 final RtpContentMap restartContentMap;
908 try {
909 if (isOffer) {
910 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
911 restartContentMap =
912 existing.modifiedCredentials(
913 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
914 } else {
915 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
916 Log.d(
917 Config.LOGTAG,
918 "received confirmation of ICE restart"
919 + newCredentials
920 + " peer_setup="
921 + setup);
922 // DTLS setup attribute needs to be rewritten to reflect current peer state
923 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
924 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
925 }
926 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
927 return isOffer;
928 } else {
929 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
930 respondWithTieBreak(jinglePacket);
931 return true;
932 }
933 } catch (final Exception exception) {
934 respondOk(jinglePacket);
935 final Throwable rootCause = Throwables.getRootCause(exception);
936 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
937 // If this happens a termination is already in progress
938 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
939 return true;
940 }
941 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
942 webRTCWrapper.close();
943 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
944 return true;
945 }
946 }
947
948 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
949 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
950 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
951 throw new IllegalStateException("Invalid peer setup");
952 }
953 return peerSetup;
954 }
955
956 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
957 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
958 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
959 }
960 this.peerDtlsSetup = setup;
961 }
962
963 private boolean applyIceRestart(
964 final JinglePacket jinglePacket,
965 final RtpContentMap restartContentMap,
966 final boolean isOffer)
967 throws ExecutionException, InterruptedException {
968 final SessionDescription sessionDescription =
969 SessionDescription.of(restartContentMap, isResponder());
970 final org.webrtc.SessionDescription.Type type =
971 isOffer
972 ? org.webrtc.SessionDescription.Type.OFFER
973 : org.webrtc.SessionDescription.Type.ANSWER;
974 org.webrtc.SessionDescription sdp =
975 new org.webrtc.SessionDescription(type, sessionDescription.toString());
976 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
977 if (isInitiator()) {
978 // We ignore the offer and respond with tie-break. This will clause the responder
979 // not to apply the content map
980 return false;
981 }
982 }
983 webRTCWrapper.setRemoteDescription(sdp).get();
984 setRemoteContentMap(restartContentMap);
985 if (isOffer) {
986 final SessionDescription localSessionDescription = setLocalSessionDescription();
987 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
988 // We need to respond OK before sending any candidates
989 respondOk(jinglePacket);
990 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
991 } else {
992 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
993 }
994 return true;
995 }
996
997 private void processCandidates(
998 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> contents) {
999 for (final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content : contents) {
1000 processCandidate(content);
1001 }
1002 }
1003
1004 private void processCandidate(
1005 final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content) {
1006 final RtpContentMap rtpContentMap = getRemoteContentMap();
1007 final List<String> indices = toIdentificationTags(rtpContentMap);
1008 final String sdpMid = content.getKey(); // aka content name
1009 final IceUdpTransportInfo transport = content.getValue().transport;
1010 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1011
1012 // TODO check that credentials remained the same
1013
1014 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1015 final String sdp;
1016 try {
1017 sdp = candidate.toSdpAttribute(credentials.ufrag);
1018 } catch (final IllegalArgumentException e) {
1019 Log.d(
1020 Config.LOGTAG,
1021 id.account.getJid().asBareJid()
1022 + ": ignoring invalid ICE candidate "
1023 + e.getMessage());
1024 continue;
1025 }
1026 final int mLineIndex = indices.indexOf(sdpMid);
1027 if (mLineIndex < 0) {
1028 Log.w(
1029 Config.LOGTAG,
1030 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1031 }
1032 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1033 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1034 this.webRTCWrapper.addIceCandidate(iceCandidate);
1035 }
1036 }
1037
1038 private RtpContentMap getRemoteContentMap() {
1039 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1040 }
1041
1042 private RtpContentMap getLocalContentMap() {
1043 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1044 }
1045
1046 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1047 final Group originalGroup = rtpContentMap.group;
1048 final List<String> identificationTags =
1049 originalGroup == null
1050 ? rtpContentMap.getNames()
1051 : originalGroup.getIdentificationTags();
1052 if (identificationTags.size() == 0) {
1053 Log.w(
1054 Config.LOGTAG,
1055 id.account.getJid().asBareJid()
1056 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1057 }
1058 return identificationTags;
1059 }
1060
1061 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1062 final JinglePacket jinglePacket, final boolean expectVerification) {
1063 try {
1064 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1065 } catch (final Exception e) {
1066 return Futures.immediateFailedFuture(e);
1067 }
1068 }
1069
1070 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1071 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1072 Log.d(
1073 Config.LOGTAG,
1074 "receiveRtpContentMap("
1075 + receivedContentMap.getClass().getSimpleName()
1076 + ",expectVerification="
1077 + expectVerification
1078 + ")");
1079 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1080 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1081 id.account
1082 .getAxolotlService()
1083 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1084 return Futures.transform(
1085 future,
1086 omemoVerifiedPayload -> {
1087 // TODO test if an exception here triggers a correct abort
1088 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1089 Log.d(
1090 Config.LOGTAG,
1091 id.account.getJid().asBareJid()
1092 + ": received verifiable DTLS fingerprint via "
1093 + omemoVerification);
1094 return omemoVerifiedPayload.getPayload();
1095 },
1096 MoreExecutors.directExecutor());
1097 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1098 return Futures.immediateFailedFuture(
1099 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1100 } else {
1101 return Futures.immediateFuture(receivedContentMap);
1102 }
1103 }
1104
1105 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1106 if (isInitiator()) {
1107 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1108 return;
1109 }
1110 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1111 Futures.addCallback(
1112 future,
1113 new FutureCallback<>() {
1114 @Override
1115 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1116 receiveSessionInitiate(jinglePacket, rtpContentMap);
1117 }
1118
1119 @Override
1120 public void onFailure(@NonNull final Throwable throwable) {
1121 respondOk(jinglePacket);
1122 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1123 }
1124 },
1125 MoreExecutors.directExecutor());
1126 }
1127
1128 private void receiveSessionInitiate(
1129 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1130 try {
1131 contentMap.requireContentDescriptions();
1132 contentMap.requireDTLSFingerprint(true);
1133 } catch (final RuntimeException e) {
1134 Log.d(
1135 Config.LOGTAG,
1136 id.account.getJid().asBareJid() + ": improperly formatted contents",
1137 Throwables.getRootCause(e));
1138 respondOk(jinglePacket);
1139 sendSessionTerminate(Reason.of(e), e.getMessage());
1140 return;
1141 }
1142 Log.d(
1143 Config.LOGTAG,
1144 "processing session-init with " + contentMap.contents.size() + " contents");
1145 final State target;
1146 if (this.state == State.PROCEED) {
1147 Preconditions.checkState(
1148 proposedMedia != null && proposedMedia.size() > 0,
1149 "proposed media must be set when processing pre-approved session-initiate");
1150 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1151 sendSessionTerminate(
1152 Reason.SECURITY_ERROR,
1153 String.format(
1154 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1155 this.proposedMedia, contentMap.getMedia()));
1156 return;
1157 }
1158 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1159 } else {
1160 target = State.SESSION_INITIALIZED;
1161 }
1162 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1163 respondOk(jinglePacket);
1164 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1165 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1166 Log.d(
1167 Config.LOGTAG,
1168 id.account.getJid().asBareJid()
1169 + ": automatically accepting session-initiate");
1170 sendSessionAccept();
1171 } else {
1172 Log.d(
1173 Config.LOGTAG,
1174 id.account.getJid().asBareJid()
1175 + ": received not pre-approved session-initiate. start ringing");
1176 startRinging();
1177 }
1178 } else {
1179 Log.d(
1180 Config.LOGTAG,
1181 String.format(
1182 "%s: received session-initiate while in state %s",
1183 id.account.getJid().asBareJid(), state));
1184 terminateWithOutOfOrder(jinglePacket);
1185 }
1186 }
1187
1188 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1189 if (isResponder()) {
1190 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1191 return;
1192 }
1193 final ListenableFuture<RtpContentMap> future =
1194 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1195 Futures.addCallback(
1196 future,
1197 new FutureCallback<>() {
1198 @Override
1199 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1200 receiveSessionAccept(jinglePacket, rtpContentMap);
1201 }
1202
1203 @Override
1204 public void onFailure(@NonNull final Throwable throwable) {
1205 respondOk(jinglePacket);
1206 Log.d(
1207 Config.LOGTAG,
1208 id.account.getJid().asBareJid()
1209 + ": improperly formatted contents in session-accept",
1210 throwable);
1211 webRTCWrapper.close();
1212 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1213 }
1214 },
1215 MoreExecutors.directExecutor());
1216 }
1217
1218 private void receiveSessionAccept(
1219 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1220 try {
1221 contentMap.requireContentDescriptions();
1222 contentMap.requireDTLSFingerprint();
1223 } catch (final RuntimeException e) {
1224 respondOk(jinglePacket);
1225 Log.d(
1226 Config.LOGTAG,
1227 id.account.getJid().asBareJid()
1228 + ": improperly formatted contents in session-accept",
1229 e);
1230 webRTCWrapper.close();
1231 sendSessionTerminate(Reason.of(e), e.getMessage());
1232 return;
1233 }
1234 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1235 if (!initiatorMedia.equals(contentMap.getMedia())) {
1236 sendSessionTerminate(
1237 Reason.SECURITY_ERROR,
1238 String.format(
1239 "Your session-included included media %s but our session-initiate was %s",
1240 this.proposedMedia, contentMap.getMedia()));
1241 return;
1242 }
1243 Log.d(
1244 Config.LOGTAG,
1245 "processing session-accept with " + contentMap.contents.size() + " contents");
1246 if (transition(State.SESSION_ACCEPTED)) {
1247 respondOk(jinglePacket);
1248 receiveSessionAccept(contentMap);
1249 } else {
1250 Log.d(
1251 Config.LOGTAG,
1252 String.format(
1253 "%s: received session-accept while in state %s",
1254 id.account.getJid().asBareJid(), state));
1255 respondOk(jinglePacket);
1256 }
1257 }
1258
1259 private void receiveSessionAccept(final RtpContentMap contentMap) {
1260 this.responderRtpContentMap = contentMap;
1261 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1262 final SessionDescription sessionDescription;
1263 try {
1264 sessionDescription = SessionDescription.of(contentMap, false);
1265 } catch (final IllegalArgumentException | NullPointerException e) {
1266 Log.d(
1267 Config.LOGTAG,
1268 id.account.getJid().asBareJid()
1269 + ": unable convert offer from session-accept to SDP",
1270 e);
1271 webRTCWrapper.close();
1272 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1273 return;
1274 }
1275 final org.webrtc.SessionDescription answer =
1276 new org.webrtc.SessionDescription(
1277 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1278 try {
1279 this.webRTCWrapper.setRemoteDescription(answer).get();
1280 } catch (final Exception e) {
1281 Log.d(
1282 Config.LOGTAG,
1283 id.account.getJid().asBareJid()
1284 + ": unable to set remote description after receiving session-accept",
1285 Throwables.getRootCause(e));
1286 webRTCWrapper.close();
1287 sendSessionTerminate(
1288 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1289 return;
1290 }
1291 processCandidates(contentMap.contents.entrySet());
1292 }
1293
1294 private void sendSessionAccept() {
1295 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1296 if (rtpContentMap == null) {
1297 throw new IllegalStateException("initiator RTP Content Map has not been set");
1298 }
1299 final SessionDescription offer;
1300 try {
1301 offer = SessionDescription.of(rtpContentMap, true);
1302 } catch (final IllegalArgumentException | NullPointerException e) {
1303 Log.d(
1304 Config.LOGTAG,
1305 id.account.getJid().asBareJid()
1306 + ": unable convert offer from session-initiate to SDP",
1307 e);
1308 webRTCWrapper.close();
1309 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1310 return;
1311 }
1312 sendSessionAccept(rtpContentMap.getMedia(), offer);
1313 }
1314
1315 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1316 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1317 }
1318
1319 private synchronized void sendSessionAccept(
1320 final Set<Media> media,
1321 final SessionDescription offer,
1322 final List<PeerConnection.IceServer> iceServers) {
1323 if (isTerminated()) {
1324 Log.w(
1325 Config.LOGTAG,
1326 id.account.getJid().asBareJid()
1327 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1328 return;
1329 }
1330 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1331 try {
1332 setupWebRTC(media, iceServers, !includeCandidates);
1333 } catch (final WebRTCWrapper.InitializationException e) {
1334 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1335 webRTCWrapper.close();
1336 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1337 return;
1338 }
1339 final org.webrtc.SessionDescription sdp =
1340 new org.webrtc.SessionDescription(
1341 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1342 try {
1343 this.webRTCWrapper.setRemoteDescription(sdp).get();
1344 addIceCandidatesFromBlackLog();
1345 org.webrtc.SessionDescription webRTCSessionDescription =
1346 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1347 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1348 } catch (final Exception e) {
1349 failureToAcceptSession(e);
1350 }
1351 }
1352
1353 private void failureToAcceptSession(final Throwable throwable) {
1354 if (isTerminated()) {
1355 return;
1356 }
1357 final Throwable rootCause = Throwables.getRootCause(throwable);
1358 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1359 webRTCWrapper.close();
1360 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1361 }
1362
1363 private void failureToPerformAction(
1364 final JinglePacket.Action action, final Throwable throwable) {
1365 if (isTerminated()) {
1366 return;
1367 }
1368 final Throwable rootCause = Throwables.getRootCause(throwable);
1369 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1370 webRTCWrapper.close();
1371 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1372 }
1373
1374 private void addIceCandidatesFromBlackLog() {
1375 Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> foo;
1376 while ((foo = this.pendingIceCandidates.poll()) != null) {
1377 processCandidate(foo);
1378 Log.d(
1379 Config.LOGTAG,
1380 id.account.getJid().asBareJid() + ": added candidate from back log");
1381 }
1382 }
1383
1384 private void prepareSessionAccept(
1385 final org.webrtc.SessionDescription webRTCSessionDescription,
1386 final boolean includeCandidates) {
1387 final SessionDescription sessionDescription =
1388 SessionDescription.parse(webRTCSessionDescription.description);
1389 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1390 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1391 if (includeCandidates) {
1392 candidates = parseCandidates(sessionDescription);
1393 } else {
1394 candidates = ImmutableMultimap.of();
1395 }
1396 this.responderRtpContentMap = respondingRtpContentMap;
1397 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1398 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1399 prepareOutgoingContentMap(respondingRtpContentMap);
1400 Futures.addCallback(
1401 outgoingContentMapFuture,
1402 new FutureCallback<>() {
1403 @Override
1404 public void onSuccess(final RtpContentMap outgoingContentMap) {
1405 if (includeCandidates) {
1406 Log.d(
1407 Config.LOGTAG,
1408 "including "
1409 + candidates.size()
1410 + " candidates in session accept");
1411 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1412 } else {
1413 sendSessionAccept(outgoingContentMap);
1414 }
1415 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1416 }
1417
1418 @Override
1419 public void onFailure(@NonNull Throwable throwable) {
1420 failureToAcceptSession(throwable);
1421 }
1422 },
1423 MoreExecutors.directExecutor());
1424 }
1425
1426 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1427 if (isTerminated()) {
1428 Log.w(
1429 Config.LOGTAG,
1430 id.account.getJid().asBareJid()
1431 + ": preparing session accept was too slow. already terminated. nothing to do.");
1432 return;
1433 }
1434 transitionOrThrow(State.SESSION_ACCEPTED);
1435 final JinglePacket sessionAccept =
1436 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1437 send(sessionAccept);
1438 }
1439
1440 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1441 final RtpContentMap rtpContentMap) {
1442 if (this.omemoVerification.hasDeviceId()) {
1443 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1444 verifiedPayloadFuture =
1445 id.account
1446 .getAxolotlService()
1447 .encrypt(
1448 rtpContentMap,
1449 id.with,
1450 omemoVerification.getDeviceId());
1451 return Futures.transform(
1452 verifiedPayloadFuture,
1453 verifiedPayload -> {
1454 omemoVerification.setOrEnsureEqual(verifiedPayload);
1455 return verifiedPayload.getPayload();
1456 },
1457 MoreExecutors.directExecutor());
1458 } else {
1459 return Futures.immediateFuture(rtpContentMap);
1460 }
1461 }
1462
1463 synchronized void deliveryMessage(
1464 final Jid from,
1465 final Element message,
1466 final String serverMessageId,
1467 final long timestamp) {
1468 Log.d(
1469 Config.LOGTAG,
1470 id.account.getJid().asBareJid()
1471 + ": delivered message to JingleRtpConnection "
1472 + message);
1473 switch (message.getName()) {
1474 case "propose" -> receivePropose(
1475 from, Propose.upgrade(message), serverMessageId, timestamp);
1476 case "proceed" -> receiveProceed(
1477 from, Proceed.upgrade(message), serverMessageId, timestamp);
1478 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1479 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1480 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1481 }
1482 }
1483
1484 void deliverFailedProceed(final String message) {
1485 Log.d(
1486 Config.LOGTAG,
1487 id.account.getJid().asBareJid()
1488 + ": receive message error for proceed message ("
1489 + Strings.nullToEmpty(message)
1490 + ")");
1491 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1492 webRTCWrapper.close();
1493 Log.d(
1494 Config.LOGTAG,
1495 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1496 this.finish();
1497 }
1498 }
1499
1500 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1501 final boolean originatedFromMyself =
1502 from.asBareJid().equals(id.account.getJid().asBareJid());
1503 if (originatedFromMyself) {
1504 if (transition(State.ACCEPTED)) {
1505 acceptedOnOtherDevice(serverMsgId, timestamp);
1506 } else {
1507 Log.d(
1508 Config.LOGTAG,
1509 id.account.getJid().asBareJid()
1510 + ": unable to transition to accept because already in state="
1511 + this.state);
1512 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1513 }
1514 } else {
1515 Log.d(
1516 Config.LOGTAG,
1517 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1518 }
1519 }
1520
1521 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1522 if (serverMsgId != null) {
1523 this.message.setServerMsgId(serverMsgId);
1524 }
1525 this.message.setTime(timestamp);
1526 this.message.setCarbon(true); // indicate that call was accepted on other device
1527 this.writeLogMessageSuccess(0);
1528 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1529 this.finish();
1530 }
1531
1532 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1533 final boolean originatedFromMyself =
1534 from.asBareJid().equals(id.account.getJid().asBareJid());
1535 // reject from another one of my clients
1536 if (originatedFromMyself) {
1537 receiveRejectFromMyself(serverMsgId, timestamp);
1538 } else if (isInitiator()) {
1539 if (from.equals(id.with)) {
1540 receiveRejectFromResponder();
1541 } else {
1542 Log.d(
1543 Config.LOGTAG,
1544 id.account.getJid()
1545 + ": ignoring reject from "
1546 + from
1547 + " for session with "
1548 + id.with);
1549 }
1550 } else {
1551 Log.d(
1552 Config.LOGTAG,
1553 id.account.getJid()
1554 + ": ignoring reject from "
1555 + from
1556 + " for session with "
1557 + id.with);
1558 }
1559 }
1560
1561 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1562 if (transition(State.REJECTED)) {
1563 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1564 this.finish();
1565 if (serverMsgId != null) {
1566 this.message.setServerMsgId(serverMsgId);
1567 }
1568 this.message.setTime(timestamp);
1569 this.message.setCarbon(true); // indicate that call was rejected on other device
1570 writeLogMessageMissed();
1571 } else {
1572 Log.d(
1573 Config.LOGTAG,
1574 "not able to transition into REJECTED because already in " + this.state);
1575 }
1576 }
1577
1578 private void receiveRejectFromResponder() {
1579 if (isInState(State.PROCEED)) {
1580 Log.d(
1581 Config.LOGTAG,
1582 id.account.getJid()
1583 + ": received reject while still in proceed. callee reconsidered");
1584 closeTransitionLogFinish(State.REJECTED_RACED);
1585 return;
1586 }
1587 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1588 Log.d(
1589 Config.LOGTAG,
1590 id.account.getJid()
1591 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1592 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1593 return;
1594 }
1595 Log.d(
1596 Config.LOGTAG,
1597 id.account.getJid()
1598 + ": ignoring reject from responder because already in state "
1599 + this.state);
1600 }
1601
1602 private void receivePropose(
1603 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1604 final boolean originatedFromMyself =
1605 from.asBareJid().equals(id.account.getJid().asBareJid());
1606 if (originatedFromMyself) {
1607 Log.d(
1608 Config.LOGTAG,
1609 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1610 } else if (transition(
1611 State.PROPOSED,
1612 () -> {
1613 final Collection<RtpDescription> descriptions =
1614 Collections2.transform(
1615 Collections2.filter(
1616 propose.getDescriptions(),
1617 d -> d instanceof RtpDescription),
1618 input -> (RtpDescription) input);
1619 final Collection<Media> media =
1620 Collections2.transform(descriptions, RtpDescription::getMedia);
1621 Preconditions.checkState(
1622 !media.contains(Media.UNKNOWN),
1623 "RTP descriptions contain unknown media");
1624 Log.d(
1625 Config.LOGTAG,
1626 id.account.getJid().asBareJid()
1627 + ": received session proposal from "
1628 + from
1629 + " for "
1630 + media);
1631 this.proposedMedia = Sets.newHashSet(media);
1632 })) {
1633 if (serverMsgId != null) {
1634 this.message.setServerMsgId(serverMsgId);
1635 }
1636 this.message.setTime(timestamp);
1637 startRinging();
1638 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1639 sendJingleMessage("ringing");
1640 }
1641 } else {
1642 Log.d(
1643 Config.LOGTAG,
1644 id.account.getJid()
1645 + ": ignoring session proposal because already in "
1646 + state);
1647 }
1648 }
1649
1650 private void startRinging() {
1651 Log.d(
1652 Config.LOGTAG,
1653 id.account.getJid().asBareJid()
1654 + ": received call from "
1655 + id.with
1656 + ". start ringing");
1657 ringingTimeoutFuture =
1658 jingleConnectionManager.schedule(
1659 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1660 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1661 }
1662
1663 private synchronized void ringingTimeout() {
1664 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1665 switch (this.state) {
1666 case PROPOSED -> {
1667 message.markUnread();
1668 rejectCallFromProposed();
1669 }
1670 case SESSION_INITIALIZED -> {
1671 message.markUnread();
1672 rejectCallFromSessionInitiate();
1673 }
1674 }
1675 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1676 }
1677
1678 private void cancelRingingTimeout() {
1679 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1680 if (future != null && !future.isCancelled()) {
1681 future.cancel(false);
1682 }
1683 }
1684
1685 private void receiveProceed(
1686 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1687 final Set<Media> media =
1688 Preconditions.checkNotNull(
1689 this.proposedMedia, "Proposed media has to be set before handling proceed");
1690 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1691 if (from.equals(id.with)) {
1692 if (isInitiator()) {
1693 if (transition(State.PROCEED)) {
1694 if (serverMsgId != null) {
1695 this.message.setServerMsgId(serverMsgId);
1696 }
1697 this.message.setTime(timestamp);
1698 final Integer remoteDeviceId = proceed.getDeviceId();
1699 if (isOmemoEnabled()) {
1700 this.omemoVerification.setDeviceId(remoteDeviceId);
1701 } else {
1702 if (remoteDeviceId != null) {
1703 Log.d(
1704 Config.LOGTAG,
1705 id.account.getJid().asBareJid()
1706 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1707 }
1708 this.omemoVerification.setDeviceId(null);
1709 }
1710 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1711 } else {
1712 Log.d(
1713 Config.LOGTAG,
1714 String.format(
1715 "%s: ignoring proceed because already in %s",
1716 id.account.getJid().asBareJid(), this.state));
1717 }
1718 } else {
1719 Log.d(
1720 Config.LOGTAG,
1721 String.format(
1722 "%s: ignoring proceed because we were not initializing",
1723 id.account.getJid().asBareJid()));
1724 }
1725 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1726 if (transition(State.ACCEPTED)) {
1727 Log.d(
1728 Config.LOGTAG,
1729 id.account.getJid().asBareJid()
1730 + ": moved session with "
1731 + id.with
1732 + " into state accepted after received carbon copied proceed");
1733 acceptedOnOtherDevice(serverMsgId, timestamp);
1734 }
1735 } else {
1736 Log.d(
1737 Config.LOGTAG,
1738 String.format(
1739 "%s: ignoring proceed from %s. was expected from %s",
1740 id.account.getJid().asBareJid(), from, id.with));
1741 }
1742 }
1743
1744 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1745 if (from.equals(id.with)) {
1746 final State target =
1747 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1748 if (transition(target)) {
1749 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1750 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1751 Log.d(
1752 Config.LOGTAG,
1753 id.account.getJid().asBareJid()
1754 + ": session with "
1755 + id.with
1756 + " has been retracted (serverMsgId="
1757 + serverMsgId
1758 + ")");
1759 if (serverMsgId != null) {
1760 this.message.setServerMsgId(serverMsgId);
1761 }
1762 this.message.setTime(timestamp);
1763 if (target == State.RETRACTED) {
1764 this.message.markUnread();
1765 }
1766 writeLogMessageMissed();
1767 finish();
1768 } else {
1769 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1770 }
1771 } else {
1772 // TODO parse retract from self
1773 Log.d(
1774 Config.LOGTAG,
1775 id.account.getJid().asBareJid()
1776 + ": received retract from "
1777 + from
1778 + ". expected retract from"
1779 + id.with
1780 + ". ignoring");
1781 }
1782 }
1783
1784 public void sendSessionInitiate() {
1785 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1786 }
1787
1788 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1789 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1790 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1791 }
1792
1793 private synchronized void sendSessionInitiate(
1794 final Set<Media> media,
1795 final State targetState,
1796 final List<PeerConnection.IceServer> iceServers) {
1797 if (isTerminated()) {
1798 Log.w(
1799 Config.LOGTAG,
1800 id.account.getJid().asBareJid()
1801 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1802 return;
1803 }
1804 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1805 try {
1806 setupWebRTC(media, iceServers, !includeCandidates);
1807 } catch (final WebRTCWrapper.InitializationException e) {
1808 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1809 webRTCWrapper.close();
1810 sendRetract(Reason.ofThrowable(e));
1811 return;
1812 }
1813 try {
1814 org.webrtc.SessionDescription webRTCSessionDescription =
1815 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1816 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1817 } catch (final Exception e) {
1818 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1819 // exceptions
1820 failureToInitiateSession(e, targetState);
1821 }
1822 }
1823
1824 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1825 if (isTerminated()) {
1826 return;
1827 }
1828 Log.d(
1829 Config.LOGTAG,
1830 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1831 Throwables.getRootCause(throwable));
1832 webRTCWrapper.close();
1833 final Reason reason = Reason.ofThrowable(throwable);
1834 if (isInState(targetState)) {
1835 sendSessionTerminate(reason, throwable.getMessage());
1836 } else {
1837 sendRetract(reason);
1838 }
1839 }
1840
1841 private void sendRetract(final Reason reason) {
1842 // TODO embed reason into retract
1843 sendJingleMessage("retract", id.with.asBareJid());
1844 transitionOrThrow(reasonToState(reason));
1845 this.finish();
1846 }
1847
1848 private void prepareSessionInitiate(
1849 final org.webrtc.SessionDescription webRTCSessionDescription,
1850 final boolean includeCandidates,
1851 final State targetState) {
1852 final SessionDescription sessionDescription =
1853 SessionDescription.parse(webRTCSessionDescription.description);
1854 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1855 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1856 if (includeCandidates) {
1857 candidates = parseCandidates(sessionDescription);
1858 } else {
1859 candidates = ImmutableMultimap.of();
1860 }
1861 this.initiatorRtpContentMap = rtpContentMap;
1862 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1863 encryptSessionInitiate(rtpContentMap);
1864 Futures.addCallback(
1865 outgoingContentMapFuture,
1866 new FutureCallback<>() {
1867 @Override
1868 public void onSuccess(final RtpContentMap outgoingContentMap) {
1869 if (includeCandidates) {
1870 Log.d(
1871 Config.LOGTAG,
1872 "including "
1873 + candidates.size()
1874 + " candidates in session initiate");
1875 sendSessionInitiate(
1876 outgoingContentMap.withCandidates(candidates), targetState);
1877 } else {
1878 sendSessionInitiate(outgoingContentMap, targetState);
1879 }
1880 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1881 }
1882
1883 @Override
1884 public void onFailure(@NonNull final Throwable throwable) {
1885 failureToInitiateSession(throwable, targetState);
1886 }
1887 },
1888 MoreExecutors.directExecutor());
1889 }
1890
1891 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1892 if (isTerminated()) {
1893 Log.w(
1894 Config.LOGTAG,
1895 id.account.getJid().asBareJid()
1896 + ": preparing session was too slow. already terminated. nothing to do.");
1897 return;
1898 }
1899 this.transitionOrThrow(targetState);
1900 final JinglePacket sessionInitiate =
1901 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1902 send(sessionInitiate);
1903 }
1904
1905 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1906 final RtpContentMap rtpContentMap) {
1907 if (this.omemoVerification.hasDeviceId()) {
1908 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1909 verifiedPayloadFuture =
1910 id.account
1911 .getAxolotlService()
1912 .encrypt(
1913 rtpContentMap,
1914 id.with,
1915 omemoVerification.getDeviceId());
1916 final ListenableFuture<RtpContentMap> future =
1917 Futures.transform(
1918 verifiedPayloadFuture,
1919 verifiedPayload -> {
1920 omemoVerification.setSessionFingerprint(
1921 verifiedPayload.getFingerprint());
1922 return verifiedPayload.getPayload();
1923 },
1924 MoreExecutors.directExecutor());
1925 if (Config.REQUIRE_RTP_VERIFICATION) {
1926 return future;
1927 }
1928 return Futures.catching(
1929 future,
1930 CryptoFailedException.class,
1931 e -> {
1932 Log.w(
1933 Config.LOGTAG,
1934 id.account.getJid().asBareJid()
1935 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1936 e);
1937 return rtpContentMap;
1938 },
1939 MoreExecutors.directExecutor());
1940 } else {
1941 return Futures.immediateFuture(rtpContentMap);
1942 }
1943 }
1944
1945 protected void sendSessionTerminate(final Reason reason) {
1946 sendSessionTerminate(reason, null);
1947 }
1948
1949
1950 protected void sendSessionTerminate(final Reason reason, final String text) {
1951 sendSessionTerminate(reason,text, this::writeLogMessage);
1952 }
1953
1954
1955 private void sendTransportInfo(
1956 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1957 final RtpContentMap transportInfo;
1958 try {
1959 final RtpContentMap rtpContentMap =
1960 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1961 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1962 } catch (final Exception e) {
1963 Log.d(
1964 Config.LOGTAG,
1965 id.account.getJid().asBareJid()
1966 + ": unable to prepare transport-info from candidate for content="
1967 + contentName);
1968 return;
1969 }
1970 final JinglePacket jinglePacket =
1971 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1972 send(jinglePacket);
1973 }
1974
1975 public RtpEndUserState getEndUserState() {
1976 switch (this.state) {
1977 case NULL, PROPOSED, SESSION_INITIALIZED -> {
1978 if (isInitiator()) {
1979 return RtpEndUserState.RINGING;
1980 } else {
1981 return RtpEndUserState.INCOMING_CALL;
1982 }
1983 }
1984 case PROCEED -> {
1985 if (isInitiator()) {
1986 return RtpEndUserState.RINGING;
1987 } else {
1988 return RtpEndUserState.ACCEPTING_CALL;
1989 }
1990 }
1991 case SESSION_INITIALIZED_PRE_APPROVED -> {
1992 if (isInitiator()) {
1993 return RtpEndUserState.RINGING;
1994 } else {
1995 return RtpEndUserState.CONNECTING;
1996 }
1997 }
1998 case SESSION_ACCEPTED -> {
1999 final ContentAddition ca = getPendingContentAddition();
2000 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2001 return RtpEndUserState.INCOMING_CONTENT_ADD;
2002 }
2003 return getPeerConnectionStateAsEndUserState();
2004 }
2005 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2006 if (isInitiator()) {
2007 return RtpEndUserState.DECLINED_OR_BUSY;
2008 } else {
2009 return RtpEndUserState.ENDED;
2010 }
2011 }
2012 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2013 return RtpEndUserState.ENDED;
2014 }
2015 case RETRACTED_RACED -> {
2016 if (isInitiator()) {
2017 return RtpEndUserState.ENDED;
2018 } else {
2019 return RtpEndUserState.RETRACTED;
2020 }
2021 }
2022 case TERMINATED_CONNECTIVITY_ERROR -> {
2023 return zeroDuration()
2024 ? RtpEndUserState.CONNECTIVITY_ERROR
2025 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2026 }
2027 case TERMINATED_APPLICATION_FAILURE -> {
2028 return RtpEndUserState.APPLICATION_ERROR;
2029 }
2030 case TERMINATED_SECURITY_ERROR -> {
2031 return RtpEndUserState.SECURITY_ERROR;
2032 }
2033 }
2034 throw new IllegalStateException(
2035 String.format("%s has no equivalent EndUserState", this.state));
2036 }
2037
2038 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2039 final PeerConnection.PeerConnectionState state;
2040 try {
2041 state = webRTCWrapper.getState();
2042 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2043 // We usually close the WebRTCWrapper *before* transitioning so we might still
2044 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2045 return RtpEndUserState.ENDING_CALL;
2046 }
2047 return switch (state) {
2048 case CONNECTED -> RtpEndUserState.CONNECTED;
2049 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2050 case CLOSED -> RtpEndUserState.ENDING_CALL;
2051 default -> zeroDuration()
2052 ? RtpEndUserState.CONNECTIVITY_ERROR
2053 : RtpEndUserState.RECONNECTING;
2054 };
2055 }
2056
2057 public ContentAddition getPendingContentAddition() {
2058 final RtpContentMap in = this.incomingContentAdd;
2059 final RtpContentMap out = this.outgoingContentAdd;
2060 if (out != null) {
2061 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2062 } else if (in != null) {
2063 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2064 } else {
2065 return null;
2066 }
2067 }
2068
2069 public Set<Media> getMedia() {
2070 final State current = getState();
2071 if (current == State.NULL) {
2072 if (isInitiator()) {
2073 return Preconditions.checkNotNull(
2074 this.proposedMedia, "RTP connection has not been initialized properly");
2075 }
2076 throw new IllegalStateException("RTP connection has not been initialized yet");
2077 }
2078 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2079 return Preconditions.checkNotNull(
2080 this.proposedMedia, "RTP connection has not been initialized properly");
2081 }
2082 final RtpContentMap localContentMap = getLocalContentMap();
2083 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2084 if (localContentMap != null) {
2085 return localContentMap.getMedia();
2086 } else if (initiatorContentMap != null) {
2087 return initiatorContentMap.getMedia();
2088 } else if (isTerminated()) {
2089 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2090 } else {
2091 return Preconditions.checkNotNull(
2092 this.proposedMedia, "RTP connection has not been initialized properly");
2093 }
2094 }
2095
2096 public boolean isVerified() {
2097 final String fingerprint = this.omemoVerification.getFingerprint();
2098 if (fingerprint == null) {
2099 return false;
2100 }
2101 final FingerprintStatus status =
2102 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2103 return status != null && status.isVerified();
2104 }
2105
2106 public boolean addMedia(final Media media) {
2107 final Set<Media> currentMedia = getMedia();
2108 if (currentMedia.contains(media)) {
2109 throw new IllegalStateException(String.format("%s has already been proposed", media));
2110 }
2111 // TODO add state protection - can only add while ACCEPTED or so
2112 Log.d(Config.LOGTAG, "adding media: " + media);
2113 return webRTCWrapper.addTrack(media);
2114 }
2115
2116 public synchronized void acceptCall() {
2117 switch (this.state) {
2118 case PROPOSED -> {
2119 cancelRingingTimeout();
2120 acceptCallFromProposed();
2121 }
2122 case SESSION_INITIALIZED -> {
2123 cancelRingingTimeout();
2124 acceptCallFromSessionInitialized();
2125 }
2126 case ACCEPTED -> Log.w(
2127 Config.LOGTAG,
2128 id.account.getJid().asBareJid()
2129 + ": the call has already been accepted with another client. UI was just lagging behind");
2130 case PROCEED, SESSION_ACCEPTED -> Log.w(
2131 Config.LOGTAG,
2132 id.account.getJid().asBareJid()
2133 + ": the call has already been accepted. user probably double tapped the UI");
2134 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2135 }
2136 }
2137
2138 public void notifyPhoneCall() {
2139 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2140 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2141 rejectCall();
2142 } else {
2143 endCall();
2144 }
2145 }
2146
2147 public synchronized void rejectCall() {
2148 if (isTerminated()) {
2149 Log.w(
2150 Config.LOGTAG,
2151 id.account.getJid().asBareJid()
2152 + ": received rejectCall() when session has already been terminated. nothing to do");
2153 return;
2154 }
2155 switch (this.state) {
2156 case PROPOSED -> rejectCallFromProposed();
2157 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2158 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2159 }
2160 }
2161
2162 public synchronized void endCall() {
2163 if (isTerminated()) {
2164 Log.w(
2165 Config.LOGTAG,
2166 id.account.getJid().asBareJid()
2167 + ": received endCall() when session has already been terminated. nothing to do");
2168 return;
2169 }
2170 if (isInState(State.PROPOSED) && isResponder()) {
2171 rejectCallFromProposed();
2172 return;
2173 }
2174 if (isInState(State.PROCEED)) {
2175 if (isInitiator()) {
2176 retractFromProceed();
2177 } else {
2178 rejectCallFromProceed();
2179 }
2180 return;
2181 }
2182 if (isInitiator()
2183 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2184 this.webRTCWrapper.close();
2185 sendSessionTerminate(Reason.CANCEL);
2186 return;
2187 }
2188 if (isInState(State.SESSION_INITIALIZED)) {
2189 rejectCallFromSessionInitiate();
2190 return;
2191 }
2192 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2193 this.webRTCWrapper.close();
2194 sendSessionTerminate(Reason.SUCCESS);
2195 return;
2196 }
2197 if (isInState(
2198 State.TERMINATED_APPLICATION_FAILURE,
2199 State.TERMINATED_CONNECTIVITY_ERROR,
2200 State.TERMINATED_DECLINED_OR_BUSY)) {
2201 Log.d(
2202 Config.LOGTAG,
2203 "ignoring request to end call because already in state " + this.state);
2204 return;
2205 }
2206 throw new IllegalStateException(
2207 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2208 }
2209
2210 private void retractFromProceed() {
2211 Log.d(Config.LOGTAG, "retract from proceed");
2212 this.sendJingleMessage("retract");
2213 closeTransitionLogFinish(State.RETRACTED_RACED);
2214 }
2215
2216 private void closeTransitionLogFinish(final State state) {
2217 this.webRTCWrapper.close();
2218 transitionOrThrow(state);
2219 writeLogMessage(state);
2220 finish();
2221 }
2222
2223 private void setupWebRTC(
2224 final Set<Media> media,
2225 final List<PeerConnection.IceServer> iceServers,
2226 final boolean trickle)
2227 throws WebRTCWrapper.InitializationException {
2228 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2229 this.webRTCWrapper.setup(
2230 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2231 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2232 }
2233
2234 private void acceptCallFromProposed() {
2235 transitionOrThrow(State.PROCEED);
2236 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2237 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2238 this.sendJingleMessage("proceed");
2239 }
2240
2241 private void rejectCallFromProposed() {
2242 transitionOrThrow(State.REJECTED);
2243 writeLogMessageMissed();
2244 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2245 this.sendJingleMessage("reject");
2246 finish();
2247 }
2248
2249 private void rejectCallFromProceed() {
2250 this.sendJingleMessage("reject");
2251 closeTransitionLogFinish(State.REJECTED_RACED);
2252 }
2253
2254 private void rejectCallFromSessionInitiate() {
2255 webRTCWrapper.close();
2256 sendSessionTerminate(Reason.DECLINE);
2257 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2258 }
2259
2260 private void sendJingleMessage(final String action) {
2261 sendJingleMessage(action, id.with);
2262 }
2263
2264 private void sendJingleMessage(final String action, final Jid to) {
2265 final MessagePacket messagePacket = new MessagePacket();
2266 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2267 messagePacket.setTo(to);
2268 final Element intent =
2269 messagePacket
2270 .addChild(action, Namespace.JINGLE_MESSAGE)
2271 .setAttribute("id", id.sessionId);
2272 if ("proceed".equals(action)) {
2273 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2274 if (isOmemoEnabled()) {
2275 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2276 final Element device =
2277 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2278 device.setAttribute("id", deviceId);
2279 }
2280 }
2281 messagePacket.addChild("store", "urn:xmpp:hints");
2282 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2283 }
2284
2285 private boolean isOmemoEnabled() {
2286 final Conversational conversational = message.getConversation();
2287 if (conversational instanceof Conversation) {
2288 return ((Conversation) conversational).getNextEncryption()
2289 == Message.ENCRYPTION_AXOLOTL;
2290 }
2291 return false;
2292 }
2293
2294 private void acceptCallFromSessionInitialized() {
2295 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2296 sendSessionAccept();
2297 }
2298
2299
2300 @Override
2301 protected synchronized boolean transition(final State target, final Runnable runnable) {
2302 if (super.transition(target, runnable)) {
2303 updateEndUserState();
2304 updateOngoingCallNotification();
2305 return true;
2306 } else {
2307 return false;
2308 }
2309 }
2310
2311 @Override
2312 public void onIceCandidate(final IceCandidate iceCandidate) {
2313 final RtpContentMap rtpContentMap =
2314 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2315 final IceUdpTransportInfo.Credentials credentials;
2316 try {
2317 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2318 } catch (final IllegalArgumentException e) {
2319 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2320 return;
2321 }
2322 final String uFrag = credentials.ufrag;
2323 final IceUdpTransportInfo.Candidate candidate =
2324 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2325 if (candidate == null) {
2326 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2327 return;
2328 }
2329 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2330 sendTransportInfo(iceCandidate.sdpMid, candidate);
2331 }
2332
2333 @Override
2334 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2335 Log.d(
2336 Config.LOGTAG,
2337 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2338 this.stateHistory.add(newState);
2339 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2340 this.sessionDuration.start();
2341 updateOngoingCallNotification();
2342 } else if (this.sessionDuration.isRunning()) {
2343 this.sessionDuration.stop();
2344 updateOngoingCallNotification();
2345 }
2346
2347 final boolean neverConnected =
2348 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2349
2350 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2351 if (neverConnected) {
2352 if (isTerminated()) {
2353 Log.d(
2354 Config.LOGTAG,
2355 id.account.getJid().asBareJid()
2356 + ": not sending session-terminate after connectivity error because session is already in state "
2357 + this.state);
2358 return;
2359 }
2360 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2361 return;
2362 } else {
2363 this.restartIce();
2364 }
2365 }
2366 updateEndUserState();
2367 }
2368
2369 private void restartIce() {
2370 this.stateHistory.clear();
2371 this.webRTCWrapper.restartIceAsync();
2372 }
2373
2374 @Override
2375 public void onRenegotiationNeeded() {
2376 this.webRTCWrapper.execute(this::renegotiate);
2377 }
2378
2379 private void renegotiate() {
2380 final SessionDescription sessionDescription;
2381 try {
2382 sessionDescription = setLocalSessionDescription();
2383 } catch (final Exception e) {
2384 final Throwable cause = Throwables.getRootCause(e);
2385 webRTCWrapper.close();
2386 if (isTerminated()) {
2387 Log.d(
2388 Config.LOGTAG,
2389 "failed to renegotiate. session was already terminated",
2390 cause);
2391 return;
2392 }
2393 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2394 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2395 return;
2396 }
2397 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2398 final RtpContentMap currentContentMap = getLocalContentMap();
2399 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2400 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2401
2402 Log.d(
2403 Config.LOGTAG,
2404 id.getAccount().getJid().asBareJid()
2405 + ": renegotiate. iceRestart="
2406 + iceRestart
2407 + " content id diff="
2408 + diff);
2409
2410 if (diff.hasModifications() && iceRestart) {
2411 webRTCWrapper.close();
2412 sendSessionTerminate(
2413 Reason.FAILED_APPLICATION,
2414 "WebRTC unexpectedly tried to modify content and transport at once");
2415 return;
2416 }
2417
2418 if (iceRestart) {
2419 initiateIceRestart(rtpContentMap);
2420 return;
2421 } else if (diff.isEmpty()) {
2422 Log.d(
2423 Config.LOGTAG,
2424 "renegotiation. nothing to do. SignalingState="
2425 + this.webRTCWrapper.getSignalingState());
2426 }
2427
2428 if (diff.added.size() > 0) {
2429 modifyLocalContentMap(rtpContentMap);
2430 sendContentAdd(rtpContentMap, diff.added);
2431 }
2432 }
2433
2434 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2435 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2436 final JinglePacket jinglePacket =
2437 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2438 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2439 jinglePacket.setTo(id.with);
2440 xmppConnectionService.sendIqPacket(
2441 id.account,
2442 jinglePacket,
2443 (account, response) -> {
2444 if (response.getType() == IqPacket.TYPE.RESULT) {
2445 Log.d(Config.LOGTAG, "received success to our ice restart");
2446 setLocalContentMap(rtpContentMap);
2447 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2448 return;
2449 }
2450 if (response.getType() == IqPacket.TYPE.ERROR) {
2451 if (isTieBreak(response)) {
2452 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2453 return;
2454 }
2455 handleIqErrorResponse(response);
2456 }
2457 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2458 handleIqTimeoutResponse(response);
2459 }
2460 });
2461 }
2462
2463 private boolean isTieBreak(final IqPacket response) {
2464 final Element error = response.findChild("error");
2465 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2466 }
2467
2468 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2469 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2470 this.outgoingContentAdd = contentAdd;
2471 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2472 prepareOutgoingContentMap(contentAdd);
2473 Futures.addCallback(
2474 outgoingContentMapFuture,
2475 new FutureCallback<>() {
2476 @Override
2477 public void onSuccess(final RtpContentMap outgoingContentMap) {
2478 sendContentAdd(outgoingContentMap);
2479 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2480 }
2481
2482 @Override
2483 public void onFailure(@NonNull Throwable throwable) {
2484 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2485 }
2486 },
2487 MoreExecutors.directExecutor());
2488 }
2489
2490 private void sendContentAdd(final RtpContentMap contentAdd) {
2491
2492 final JinglePacket jinglePacket =
2493 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2494 jinglePacket.setTo(id.with);
2495 xmppConnectionService.sendIqPacket(
2496 id.account,
2497 jinglePacket,
2498 (connection, response) -> {
2499 if (response.getType() == IqPacket.TYPE.RESULT) {
2500 Log.d(
2501 Config.LOGTAG,
2502 id.getAccount().getJid().asBareJid()
2503 + ": received ACK to our content-add");
2504 return;
2505 }
2506 if (response.getType() == IqPacket.TYPE.ERROR) {
2507 if (isTieBreak(response)) {
2508 this.outgoingContentAdd = null;
2509 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2510 return;
2511 }
2512 handleIqErrorResponse(response);
2513 }
2514 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2515 handleIqTimeoutResponse(response);
2516 }
2517 });
2518 }
2519
2520 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2521 if (isInitiator()) {
2522 this.initiatorRtpContentMap = rtpContentMap;
2523 } else {
2524 this.responderRtpContentMap = rtpContentMap;
2525 }
2526 }
2527
2528 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2529 if (isInitiator()) {
2530 this.responderRtpContentMap = rtpContentMap;
2531 } else {
2532 this.initiatorRtpContentMap = rtpContentMap;
2533 }
2534 }
2535
2536 // this method is to be used for content map modifications that modify media
2537 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2538 final RtpContentMap activeContents = rtpContentMap.activeContents();
2539 setLocalContentMap(activeContents);
2540 this.webRTCWrapper.switchSpeakerPhonePreference(
2541 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2542 updateEndUserState();
2543 }
2544
2545 private SessionDescription setLocalSessionDescription()
2546 throws ExecutionException, InterruptedException {
2547 final org.webrtc.SessionDescription sessionDescription =
2548 this.webRTCWrapper.setLocalDescription(false).get();
2549 return SessionDescription.parse(sessionDescription.description);
2550 }
2551
2552 private void closeWebRTCSessionAfterFailedConnection() {
2553 this.webRTCWrapper.close();
2554 synchronized (this) {
2555 if (isTerminated()) {
2556 Log.d(
2557 Config.LOGTAG,
2558 id.account.getJid().asBareJid()
2559 + ": no need to send session-terminate after failed connection. Other party already did");
2560 return;
2561 }
2562 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2563 }
2564 }
2565
2566 public boolean zeroDuration() {
2567 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2568 }
2569
2570 public long getCallDuration() {
2571 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2572 }
2573
2574 public AppRTCAudioManager getAudioManager() {
2575 return webRTCWrapper.getAudioManager();
2576 }
2577
2578 public boolean isMicrophoneEnabled() {
2579 return webRTCWrapper.isMicrophoneEnabled();
2580 }
2581
2582 public boolean setMicrophoneEnabled(final boolean enabled) {
2583 return webRTCWrapper.setMicrophoneEnabled(enabled);
2584 }
2585
2586 public boolean isVideoEnabled() {
2587 return webRTCWrapper.isVideoEnabled();
2588 }
2589
2590 public void setVideoEnabled(final boolean enabled) {
2591 webRTCWrapper.setVideoEnabled(enabled);
2592 }
2593
2594 public boolean isCameraSwitchable() {
2595 return webRTCWrapper.isCameraSwitchable();
2596 }
2597
2598 public boolean isFrontCamera() {
2599 return webRTCWrapper.isFrontCamera();
2600 }
2601
2602 public ListenableFuture<Boolean> switchCamera() {
2603 return webRTCWrapper.switchCamera();
2604 }
2605
2606 @Override
2607 public void onAudioDeviceChanged(
2608 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2609 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2610 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2611 selectedAudioDevice, availableAudioDevices);
2612 }
2613
2614 private void updateEndUserState() {
2615 final RtpEndUserState endUserState = getEndUserState();
2616 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2617 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2618 id.account, id.with, id.sessionId, endUserState);
2619 }
2620
2621 private void updateOngoingCallNotification() {
2622 final State state = this.state;
2623 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2624 final boolean reconnecting;
2625 if (state == State.SESSION_ACCEPTED) {
2626 reconnecting =
2627 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2628 } else {
2629 reconnecting = false;
2630 }
2631 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2632 } else {
2633 xmppConnectionService.removeOngoingCall();
2634 }
2635 }
2636
2637 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2638 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2639 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2640 request.setTo(id.account.getDomain());
2641 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2642 xmppConnectionService.sendIqPacket(
2643 id.account,
2644 request,
2645 (account, response) -> {
2646 final var iceServers = IceServers.parse(response);
2647 if (iceServers.size() == 0) {
2648 Log.w(
2649 Config.LOGTAG,
2650 id.account.getJid().asBareJid()
2651 + ": no ICE server found "
2652 + response);
2653 }
2654 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2655 });
2656 } else {
2657 Log.w(
2658 Config.LOGTAG,
2659 id.account.getJid().asBareJid() + ": has no external service discovery");
2660 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2661 }
2662 }
2663
2664 @Override
2665 protected void terminateTransport() {
2666 this.webRTCWrapper.close();
2667 }
2668
2669 @Override
2670 protected void finish() {
2671 if (isTerminated()) {
2672 this.cancelRingingTimeout();
2673 this.webRTCWrapper.verifyClosed();
2674 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2675 super.finish();
2676 } else {
2677 throw new IllegalStateException(
2678 String.format("Unable to call finish from %s", this.state));
2679 }
2680 }
2681
2682 private void writeLogMessage(final State state) {
2683 final long duration = getCallDuration();
2684 if (state == State.TERMINATED_SUCCESS
2685 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2686 writeLogMessageSuccess(duration);
2687 } else {
2688 writeLogMessageMissed();
2689 }
2690 }
2691
2692 private void writeLogMessageSuccess(final long duration) {
2693 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2694 this.writeMessage();
2695 }
2696
2697 private void writeLogMessageMissed() {
2698 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2699 this.writeMessage();
2700 }
2701
2702 private void writeMessage() {
2703 final Conversational conversational = message.getConversation();
2704 if (conversational instanceof Conversation) {
2705 ((Conversation) conversational).add(this.message);
2706 xmppConnectionService.createMessageAsync(message);
2707 xmppConnectionService.updateConversationUi();
2708 } else {
2709 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2710 }
2711 }
2712
2713 public Optional<VideoTrack> getLocalVideoTrack() {
2714 return webRTCWrapper.getLocalVideoTrack();
2715 }
2716
2717 public Optional<VideoTrack> getRemoteVideoTrack() {
2718 return webRTCWrapper.getRemoteVideoTrack();
2719 }
2720
2721 public EglBase.Context getEglBaseContext() {
2722 return webRTCWrapper.getEglBaseContext();
2723 }
2724
2725 void setProposedMedia(final Set<Media> media) {
2726 this.proposedMedia = media;
2727 }
2728
2729 public void fireStateUpdate() {
2730 final RtpEndUserState endUserState = getEndUserState();
2731 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2732 id.account, id.with, id.sessionId, endUserState);
2733 }
2734
2735 public boolean isSwitchToVideoAvailable() {
2736 final boolean prerequisite =
2737 Media.audioOnly(getMedia())
2738 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2739 .contains(getEndUserState());
2740 return prerequisite && remoteHasVideoFeature();
2741 }
2742
2743 private boolean remoteHasVideoFeature() {
2744 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2745 }
2746
2747 private boolean remoteHasSdpOfferAnswer() {
2748 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2749 }
2750
2751 private interface OnIceServersDiscovered {
2752 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2753 }
2754}