1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7
8import androidx.annotation.NonNull;
9import androidx.annotation.Nullable;
10
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.AppRTCAudioManager;
38import eu.siacs.conversations.services.CallIntegration;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
50import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
51import eu.siacs.conversations.xmpp.stanzas.IqPacket;
52import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
53
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.util.Arrays;
60import java.util.Collection;
61import java.util.Collections;
62import java.util.LinkedList;
63import java.util.List;
64import java.util.Map;
65import java.util.Queue;
66import java.util.Set;
67import java.util.concurrent.ExecutionException;
68import java.util.concurrent.ScheduledFuture;
69import java.util.concurrent.TimeUnit;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback, CallIntegration.Callback {
73
74 public static final List<State> STATES_SHOWING_ONGOING_CALL =
75 Arrays.asList(
76 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
77 private static final long BUSY_TIME_OUT = 30;
78
79 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
80 private final Queue<
81 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
82 pendingIceCandidates = new LinkedList<>();
83 private final OmemoVerification omemoVerification = new OmemoVerification();
84 private final CallIntegration callIntegration;
85 private final Message message;
86
87 private Set<Media> proposedMedia;
88 private RtpContentMap initiatorRtpContentMap;
89 private RtpContentMap responderRtpContentMap;
90 private RtpContentMap incomingContentAdd;
91 private RtpContentMap outgoingContentAdd;
92 private IceUdpTransportInfo.Setup peerDtlsSetup;
93 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
94 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
95 private ScheduledFuture<?> ringingTimeoutFuture;
96
97 JingleRtpConnection(
98 final JingleConnectionManager jingleConnectionManager,
99 final Id id,
100 final Jid initiator) {
101 this(
102 jingleConnectionManager,
103 id,
104 initiator,
105 new CallIntegration(
106 jingleConnectionManager
107 .getXmppConnectionService()
108 .getApplicationContext()));
109 this.callIntegration.setAddress(
110 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
111 this.callIntegration.setInitialized();
112 }
113
114 JingleRtpConnection(
115 final JingleConnectionManager jingleConnectionManager,
116 final Id id,
117 final Jid initiator,
118 final CallIntegration callIntegration) {
119 super(jingleConnectionManager, id, initiator);
120 final Conversation conversation =
121 jingleConnectionManager
122 .getXmppConnectionService()
123 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
124 this.message =
125 new Message(
126 conversation,
127 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
128 Message.TYPE_RTP_SESSION,
129 id.sessionId);
130 this.callIntegration = callIntegration;
131 this.callIntegration.setCallback(this);
132 }
133
134 @Override
135 synchronized void deliverPacket(final JinglePacket jinglePacket) {
136 switch (jinglePacket.getAction()) {
137 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
138 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
139 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
140 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
141 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
142 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
143 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
144 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
145 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
146 default -> {
147 respondOk(jinglePacket);
148 Log.d(
149 Config.LOGTAG,
150 String.format(
151 "%s: received unhandled jingle action %s",
152 id.account.getJid().asBareJid(), jinglePacket.getAction()));
153 }
154 }
155 }
156
157 @Override
158 synchronized void notifyRebound() {
159 if (isTerminated()) {
160 return;
161 }
162 webRTCWrapper.close();
163 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
164 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
165 }
166 if (isInState(
167 State.SESSION_INITIALIZED,
168 State.SESSION_INITIALIZED_PRE_APPROVED,
169 State.SESSION_ACCEPTED)) {
170 // we might have already changed resources (full jid) at this point; so this might not
171 // even reach the other party
172 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
173 } else {
174 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
175 finish();
176 }
177 }
178
179 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
180 respondOk(jinglePacket);
181 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
182 final State previous = this.state;
183 Log.d(
184 Config.LOGTAG,
185 id.account.getJid().asBareJid()
186 + ": received session terminate reason="
187 + wrapper.reason
188 + "("
189 + Strings.nullToEmpty(wrapper.text)
190 + ") while in state "
191 + previous);
192 if (TERMINATED.contains(previous)) {
193 Log.d(
194 Config.LOGTAG,
195 id.account.getJid().asBareJid()
196 + ": ignoring session terminate because already in "
197 + previous);
198 return;
199 }
200 webRTCWrapper.close();
201 final State target = reasonToState(wrapper.reason);
202 transitionOrThrow(target);
203 writeLogMessage(target);
204 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
205 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
206 }
207 finish();
208 }
209
210 private void receiveTransportInfo(final JinglePacket jinglePacket) {
211 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
212 // INITIALIZED only after transport-info has been received
213 if (isInState(
214 State.NULL,
215 State.PROCEED,
216 State.SESSION_INITIALIZED,
217 State.SESSION_INITIALIZED_PRE_APPROVED,
218 State.SESSION_ACCEPTED)) {
219 final RtpContentMap contentMap;
220 try {
221 contentMap = RtpContentMap.of(jinglePacket);
222 } catch (final IllegalArgumentException | NullPointerException e) {
223 Log.d(
224 Config.LOGTAG,
225 id.account.getJid().asBareJid()
226 + ": improperly formatted contents; ignoring",
227 e);
228 respondOk(jinglePacket);
229 return;
230 }
231 receiveTransportInfo(jinglePacket, contentMap);
232 } else {
233 if (isTerminated()) {
234 respondOk(jinglePacket);
235 Log.d(
236 Config.LOGTAG,
237 id.account.getJid().asBareJid()
238 + ": ignoring out-of-order transport info; we where already terminated");
239 } else {
240 Log.d(
241 Config.LOGTAG,
242 id.account.getJid().asBareJid()
243 + ": received transport info while in state="
244 + this.state);
245 terminateWithOutOfOrder(jinglePacket);
246 }
247 }
248 }
249
250 private void receiveTransportInfo(
251 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
252 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
253 candidates = contentMap.contents.entrySet();
254 final RtpContentMap remote = getRemoteContentMap();
255 final Set<String> remoteContentIds =
256 remote == null ? Collections.emptySet() : remote.contents.keySet();
257 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
258 Log.d(
259 Config.LOGTAG,
260 "received transport-info for unknown contents "
261 + contentMap.contents.keySet()
262 + " (known: "
263 + remoteContentIds
264 + ")");
265 respondOk(jinglePacket);
266 pendingIceCandidates.addAll(candidates);
267 return;
268 }
269 if (this.state != State.SESSION_ACCEPTED) {
270 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
271 respondOk(jinglePacket);
272 pendingIceCandidates.addAll(candidates);
273 return;
274 }
275 // zero candidates + modified credentials are an ICE restart offer
276 if (checkForIceRestart(jinglePacket, contentMap)) {
277 return;
278 }
279 respondOk(jinglePacket);
280 try {
281 processCandidates(candidates);
282 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
283 Log.w(
284 Config.LOGTAG,
285 id.account.getJid().asBareJid()
286 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
287 }
288 }
289
290 private void receiveContentAdd(final JinglePacket jinglePacket) {
291 final RtpContentMap modification;
292 try {
293 modification = RtpContentMap.of(jinglePacket);
294 modification.requireContentDescriptions();
295 } catch (final RuntimeException e) {
296 Log.d(
297 Config.LOGTAG,
298 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
299 Throwables.getRootCause(e));
300 respondOk(jinglePacket);
301 webRTCWrapper.close();
302 sendSessionTerminate(Reason.of(e), e.getMessage());
303 return;
304 }
305 if (isInState(State.SESSION_ACCEPTED)) {
306 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
307 final ListenableFuture<RtpContentMap> future =
308 receiveRtpContentMap(
309 modification,
310 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
311 Futures.addCallback(
312 future,
313 new FutureCallback<>() {
314 @Override
315 public void onSuccess(final RtpContentMap rtpContentMap) {
316 receiveContentAdd(jinglePacket, rtpContentMap);
317 }
318
319 @Override
320 public void onFailure(@NonNull Throwable throwable) {
321 respondOk(jinglePacket);
322 final Throwable rootCause = Throwables.getRootCause(throwable);
323 Log.d(
324 Config.LOGTAG,
325 id.account.getJid().asBareJid()
326 + ": improperly formatted contents in content-add",
327 throwable);
328 webRTCWrapper.close();
329 sendSessionTerminate(
330 Reason.ofThrowable(rootCause), rootCause.getMessage());
331 }
332 },
333 MoreExecutors.directExecutor());
334 } else {
335 terminateWithOutOfOrder(jinglePacket);
336 }
337 }
338
339 private void receiveContentAdd(
340 final JinglePacket jinglePacket, final RtpContentMap modification) {
341 final RtpContentMap remote = getRemoteContentMap();
342 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
343 respondOk(jinglePacket);
344 this.webRTCWrapper.close();
345 sendSessionTerminate(
346 Reason.FAILED_APPLICATION,
347 String.format(
348 "contents with names %s already exists",
349 Joiner.on(", ").join(modification.getNames())));
350 return;
351 }
352 final ContentAddition contentAddition =
353 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
354
355 final RtpContentMap outgoing = this.outgoingContentAdd;
356 final Set<ContentAddition.Summary> outgoingContentAddSummary =
357 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
358
359 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
360 if (isInitiator()) {
361 Log.d(
362 Config.LOGTAG,
363 id.getAccount().getJid().asBareJid()
364 + ": respond with tie break to matching content-add offer");
365 respondWithTieBreak(jinglePacket);
366 } else {
367 Log.d(
368 Config.LOGTAG,
369 id.getAccount().getJid().asBareJid()
370 + ": automatically accept matching content-add offer");
371 acceptContentAdd(contentAddition.summary, modification);
372 }
373 return;
374 }
375
376 // once we can display multiple video tracks we can be more loose with this condition
377 // theoretically it should also be fine to automatically accept audio only contents
378 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
379 Log.d(
380 Config.LOGTAG,
381 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
382 this.incomingContentAdd = modification;
383 respondOk(jinglePacket);
384 updateEndUserState();
385 } else {
386 respondOk(jinglePacket);
387 // TODO do we want to add a reason?
388 rejectContentAdd(modification);
389 }
390 }
391
392 private void receiveContentAccept(final JinglePacket jinglePacket) {
393 final RtpContentMap receivedContentAccept;
394 try {
395 receivedContentAccept = RtpContentMap.of(jinglePacket);
396 receivedContentAccept.requireContentDescriptions();
397 } catch (final RuntimeException e) {
398 Log.d(
399 Config.LOGTAG,
400 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
401 Throwables.getRootCause(e));
402 respondOk(jinglePacket);
403 webRTCWrapper.close();
404 sendSessionTerminate(Reason.of(e), e.getMessage());
405 return;
406 }
407
408 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
409 if (outgoingContentAdd == null) {
410 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
411 terminateWithOutOfOrder(jinglePacket);
412 return;
413 }
414 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
415 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
416 this.outgoingContentAdd = null;
417 respondOk(jinglePacket);
418 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
419 final ListenableFuture<RtpContentMap> future =
420 receiveRtpContentMap(
421 receivedContentAccept,
422 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
423 Futures.addCallback(
424 future,
425 new FutureCallback<>() {
426 @Override
427 public void onSuccess(final RtpContentMap result) {
428 receiveContentAccept(result);
429 }
430
431 @Override
432 public void onFailure(@NonNull final Throwable throwable) {
433 webRTCWrapper.close();
434 sendSessionTerminate(
435 Reason.ofThrowable(throwable), throwable.getMessage());
436 }
437 },
438 MoreExecutors.directExecutor());
439 } else {
440 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
441 terminateWithOutOfOrder(jinglePacket);
442 }
443 }
444
445 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
446 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
447 final RtpContentMap modifiedContentMap =
448 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
449
450 setRemoteContentMap(modifiedContentMap);
451
452 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
453
454 final org.webrtc.SessionDescription sdp =
455 new org.webrtc.SessionDescription(
456 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
457
458 try {
459 this.webRTCWrapper.setRemoteDescription(sdp).get();
460 } catch (final Exception e) {
461 final Throwable cause = Throwables.getRootCause(e);
462 Log.d(
463 Config.LOGTAG,
464 id.getAccount().getJid().asBareJid()
465 + ": unable to set remote description after receiving content-accept",
466 cause);
467 webRTCWrapper.close();
468 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
469 return;
470 }
471 Log.d(
472 Config.LOGTAG,
473 id.getAccount().getJid().asBareJid()
474 + ": remote has accepted content-add "
475 + ContentAddition.summary(receivedContentAccept));
476 processCandidates(receivedContentAccept.contents.entrySet());
477 updateEndUserState();
478 }
479
480 private void receiveContentModify(final JinglePacket jinglePacket) {
481 if (this.state != State.SESSION_ACCEPTED) {
482 terminateWithOutOfOrder(jinglePacket);
483 return;
484 }
485 final Map<String, Content.Senders> modification =
486 Maps.transformEntries(
487 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
488 final boolean isInitiator = isInitiator();
489 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
490 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
491 final Set<String> currentOutgoingMediaIds =
492 currentOutgoing == null
493 ? Collections.emptySet()
494 : currentOutgoing.contents.keySet();
495 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
496 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
497 respondOk(jinglePacket);
498 final RtpContentMap modifiedContentMap;
499 try {
500 modifiedContentMap =
501 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
502 } catch (final IllegalArgumentException e) {
503 webRTCWrapper.close();
504 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
505 return;
506 }
507 this.outgoingContentAdd = modifiedContentMap;
508 Log.d(
509 Config.LOGTAG,
510 id.account.getJid().asBareJid()
511 + ": processed content-modification for pending content-add");
512 } else if (remoteContentMap != null
513 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
514 respondOk(jinglePacket);
515 final RtpContentMap modifiedRemoteContentMap;
516 try {
517 modifiedRemoteContentMap =
518 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
519 } catch (final IllegalArgumentException e) {
520 webRTCWrapper.close();
521 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
522 return;
523 }
524 final SessionDescription offer;
525 try {
526 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
527 } catch (final IllegalArgumentException | NullPointerException e) {
528 Log.d(
529 Config.LOGTAG,
530 id.getAccount().getJid().asBareJid()
531 + ": unable convert offer from content-modify to SDP",
532 e);
533 webRTCWrapper.close();
534 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
535 return;
536 }
537 Log.d(
538 Config.LOGTAG,
539 id.account.getJid().asBareJid() + ": auto accepting content-modification");
540 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
541 } else {
542 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
543 respondWithItemNotFound(jinglePacket);
544 }
545 }
546
547 private void autoAcceptContentModify(
548 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
549 this.setRemoteContentMap(modifiedRemoteContentMap);
550 final org.webrtc.SessionDescription sdp =
551 new org.webrtc.SessionDescription(
552 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
553 try {
554 this.webRTCWrapper.setRemoteDescription(sdp).get();
555 // auto accept is only done when we already have tracks
556 final SessionDescription answer = setLocalSessionDescription();
557 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
558 modifyLocalContentMap(rtpContentMap);
559 // we do not need to send an answer but do we have to resend the candidates currently in
560 // SDP?
561 // resendCandidatesFromSdp(answer);
562 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
563 } catch (final Exception e) {
564 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
565 webRTCWrapper.close();
566 sendSessionTerminate(Reason.FAILED_APPLICATION);
567 }
568 }
569
570 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
571 final SessionDescription answer) {
572 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
573 new ImmutableMultimap.Builder<>();
574 for (final SessionDescription.Media media : answer.media) {
575 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
576 if (Strings.isNullOrEmpty(mid)) {
577 continue;
578 }
579 for (final String sdpCandidate : media.attributes.get("candidate")) {
580 final IceUdpTransportInfo.Candidate candidate =
581 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
582 if (candidate != null) {
583 candidateBuilder.put(mid, candidate);
584 }
585 }
586 }
587 return candidateBuilder.build();
588 }
589
590 private void receiveContentReject(final JinglePacket jinglePacket) {
591 final RtpContentMap receivedContentReject;
592 try {
593 receivedContentReject = RtpContentMap.of(jinglePacket);
594 } catch (final RuntimeException e) {
595 Log.d(
596 Config.LOGTAG,
597 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
598 Throwables.getRootCause(e));
599 respondOk(jinglePacket);
600 this.webRTCWrapper.close();
601 sendSessionTerminate(Reason.of(e), e.getMessage());
602 return;
603 }
604
605 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
606 if (outgoingContentAdd == null) {
607 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
608 terminateWithOutOfOrder(jinglePacket);
609 return;
610 }
611 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
612 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
613 this.outgoingContentAdd = null;
614 respondOk(jinglePacket);
615 Log.d(Config.LOGTAG, jinglePacket.toString());
616 receiveContentReject(ourSummary);
617 } else {
618 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
619 terminateWithOutOfOrder(jinglePacket);
620 }
621 }
622
623 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
624 try {
625 this.webRTCWrapper.removeTrack(Media.VIDEO);
626 final RtpContentMap localContentMap = customRollback();
627 modifyLocalContentMap(localContentMap);
628 } catch (final Exception e) {
629 final Throwable cause = Throwables.getRootCause(e);
630 Log.d(
631 Config.LOGTAG,
632 id.getAccount().getJid().asBareJid()
633 + ": unable to rollback local description after receiving content-reject",
634 cause);
635 webRTCWrapper.close();
636 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
637 return;
638 }
639 Log.d(
640 Config.LOGTAG,
641 id.getAccount().getJid().asBareJid()
642 + ": remote has rejected our content-add "
643 + summary);
644 }
645
646 private void receiveContentRemove(final JinglePacket jinglePacket) {
647 final RtpContentMap receivedContentRemove;
648 try {
649 receivedContentRemove = RtpContentMap.of(jinglePacket);
650 receivedContentRemove.requireContentDescriptions();
651 } catch (final RuntimeException e) {
652 Log.d(
653 Config.LOGTAG,
654 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
655 Throwables.getRootCause(e));
656 respondOk(jinglePacket);
657 this.webRTCWrapper.close();
658 sendSessionTerminate(Reason.of(e), e.getMessage());
659 return;
660 }
661 respondOk(jinglePacket);
662 receiveContentRemove(receivedContentRemove);
663 }
664
665 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
666 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
667 final Set<ContentAddition.Summary> contentAddSummary =
668 incomingContentAdd == null
669 ? Collections.emptySet()
670 : ContentAddition.summary(incomingContentAdd);
671 final Set<ContentAddition.Summary> removeSummary =
672 ContentAddition.summary(receivedContentRemove);
673 if (contentAddSummary.equals(removeSummary)) {
674 this.incomingContentAdd = null;
675 updateEndUserState();
676 } else {
677 webRTCWrapper.close();
678 sendSessionTerminate(
679 Reason.FAILED_APPLICATION,
680 String.format(
681 "%s only supports %s as a means to retract a not yet accepted %s",
682 BuildConfig.APP_NAME,
683 JinglePacket.Action.CONTENT_REMOVE,
684 JinglePacket.Action.CONTENT_ADD));
685 }
686 }
687
688 public synchronized void retractContentAdd() {
689 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
690 if (outgoingContentAdd == null) {
691 throw new IllegalStateException("Not outgoing content add");
692 }
693 try {
694 webRTCWrapper.removeTrack(Media.VIDEO);
695 final RtpContentMap localContentMap = customRollback();
696 modifyLocalContentMap(localContentMap);
697 } catch (final Exception e) {
698 final Throwable cause = Throwables.getRootCause(e);
699 Log.d(
700 Config.LOGTAG,
701 id.getAccount().getJid().asBareJid()
702 + ": unable to rollback local description after trying to retract content-add",
703 cause);
704 webRTCWrapper.close();
705 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
706 return;
707 }
708 this.outgoingContentAdd = null;
709 final JinglePacket retract =
710 outgoingContentAdd
711 .toStub()
712 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
713 this.send(retract);
714 Log.d(
715 Config.LOGTAG,
716 id.getAccount().getJid()
717 + ": retract content-add "
718 + ContentAddition.summary(outgoingContentAdd));
719 }
720
721 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
722 final SessionDescription sdp = setLocalSessionDescription();
723 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
724 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
725 this.webRTCWrapper
726 .setRemoteDescription(
727 new org.webrtc.SessionDescription(
728 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
729 .get();
730 return localRtpContentMap;
731 }
732
733 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
734 final RtpContentMap currentRemote = getRemoteContentMap();
735 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
736 if (diff.isEmpty()) {
737 throw new IllegalStateException(
738 "Unexpected rollback condition. No difference between local and remote");
739 }
740 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
741 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
742 final RtpContentMap nextRemote =
743 currentRemote.addContent(
744 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
745 return SessionDescription.of(nextRemote, isResponder());
746 }
747 throw new IllegalStateException(
748 "Unexpected rollback condition. Senders were not uniformly none");
749 }
750
751 public synchronized void acceptContentAdd(
752 @NonNull final Set<ContentAddition.Summary> contentAddition) {
753 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
754 if (incomingContentAdd == null) {
755 throw new IllegalStateException("No incoming content add");
756 }
757
758 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
759 this.incomingContentAdd = null;
760 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
761 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
762 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
763 Log.d(
764 Config.LOGTAG,
765 "content addition is receive only. we want to upgrade to 'both'");
766 final RtpContentMap modifiedSenders =
767 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
768 final JinglePacket proposedContentModification =
769 modifiedSenders
770 .toStub()
771 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
772 proposedContentModification.setTo(id.with);
773 xmppConnectionService.sendIqPacket(
774 id.account,
775 proposedContentModification,
776 (account, response) -> {
777 if (response.getType() == IqPacket.TYPE.RESULT) {
778 Log.d(
779 Config.LOGTAG,
780 id.account.getJid().asBareJid()
781 + ": remote has accepted our upgrade to senders=both");
782 acceptContentAdd(
783 ContentAddition.summary(modifiedSenders), modifiedSenders);
784 } else {
785 Log.d(
786 Config.LOGTAG,
787 id.account.getJid().asBareJid()
788 + ": remote has rejected our upgrade to senders=both");
789 acceptContentAdd(contentAddition, incomingContentAdd);
790 }
791 });
792 } else {
793 acceptContentAdd(contentAddition, incomingContentAdd);
794 }
795 } else {
796 throw new IllegalStateException(
797 "Accepted content add does not match pending content-add");
798 }
799 }
800
801 private void acceptContentAdd(
802 @NonNull final Set<ContentAddition.Summary> contentAddition,
803 final RtpContentMap incomingContentAdd) {
804 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
805 final RtpContentMap modifiedContentMap =
806 getRemoteContentMap().addContent(incomingContentAdd, setup);
807 this.setRemoteContentMap(modifiedContentMap);
808
809 final SessionDescription offer;
810 try {
811 offer = SessionDescription.of(modifiedContentMap, isResponder());
812 } catch (final IllegalArgumentException | NullPointerException e) {
813 Log.d(
814 Config.LOGTAG,
815 id.getAccount().getJid().asBareJid()
816 + ": unable convert offer from content-add to SDP",
817 e);
818 webRTCWrapper.close();
819 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
820 return;
821 }
822 this.incomingContentAdd = null;
823 acceptContentAdd(contentAddition, offer);
824 }
825
826 private void acceptContentAdd(
827 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
828 final org.webrtc.SessionDescription sdp =
829 new org.webrtc.SessionDescription(
830 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
831 try {
832 this.webRTCWrapper.setRemoteDescription(sdp).get();
833
834 // TODO add tracks for 'media' where contentAddition.senders matches
835
836 // TODO if senders.sending(isInitiator())
837
838 this.webRTCWrapper.addTrack(Media.VIDEO);
839
840 // TODO add additional transceivers for recv only cases
841
842 final SessionDescription answer = setLocalSessionDescription();
843 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
844
845 final RtpContentMap contentAcceptMap =
846 rtpContentMap.toContentModification(
847 Collections2.transform(contentAddition, ca -> ca.name));
848
849 Log.d(
850 Config.LOGTAG,
851 id.getAccount().getJid().asBareJid()
852 + ": sending content-accept "
853 + ContentAddition.summary(contentAcceptMap));
854
855 addIceCandidatesFromBlackLog();
856
857 modifyLocalContentMap(rtpContentMap);
858 final ListenableFuture<RtpContentMap> future =
859 prepareOutgoingContentMap(contentAcceptMap);
860 Futures.addCallback(
861 future,
862 new FutureCallback<>() {
863 @Override
864 public void onSuccess(final RtpContentMap rtpContentMap) {
865 sendContentAccept(rtpContentMap);
866 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
867 }
868
869 @Override
870 public void onFailure(@NonNull final Throwable throwable) {
871 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
872 }
873 },
874 MoreExecutors.directExecutor());
875 } catch (final Exception e) {
876 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
877 webRTCWrapper.close();
878 sendSessionTerminate(Reason.FAILED_APPLICATION);
879 }
880 }
881
882 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
883 final JinglePacket jinglePacket =
884 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
885 send(jinglePacket);
886 }
887
888 public synchronized void rejectContentAdd() {
889 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
890 if (incomingContentAdd == null) {
891 throw new IllegalStateException("No incoming content add");
892 }
893 this.incomingContentAdd = null;
894 updateEndUserState();
895 rejectContentAdd(incomingContentAdd);
896 }
897
898 private void rejectContentAdd(final RtpContentMap contentMap) {
899 final JinglePacket jinglePacket =
900 contentMap
901 .toStub()
902 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
903 Log.d(
904 Config.LOGTAG,
905 id.getAccount().getJid().asBareJid()
906 + ": rejecting content "
907 + ContentAddition.summary(contentMap));
908 send(jinglePacket);
909 }
910
911 private boolean checkForIceRestart(
912 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
913 final RtpContentMap existing = getRemoteContentMap();
914 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
915 final IceUdpTransportInfo.Credentials newCredentials;
916 try {
917 existingCredentials = existing.getCredentials();
918 newCredentials = rtpContentMap.getDistinctCredentials();
919 } catch (final IllegalStateException e) {
920 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
921 return false;
922 }
923 if (existingCredentials.contains(newCredentials)) {
924 return false;
925 }
926 // TODO an alternative approach is to check if we already got an iq result to our
927 // ICE-restart
928 // and if that's the case we are seeing an answer.
929 // This might be more spec compliant but also more error prone potentially
930 final boolean isSignalStateStable =
931 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
932 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
933 final boolean isOffer = rtpContentMap.emptyCandidates();
934 final RtpContentMap restartContentMap;
935 try {
936 if (isOffer) {
937 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
938 restartContentMap =
939 existing.modifiedCredentials(
940 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
941 } else {
942 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
943 Log.d(
944 Config.LOGTAG,
945 "received confirmation of ICE restart"
946 + newCredentials
947 + " peer_setup="
948 + setup);
949 // DTLS setup attribute needs to be rewritten to reflect current peer state
950 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
951 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
952 }
953 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
954 return isOffer;
955 } else {
956 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
957 respondWithTieBreak(jinglePacket);
958 return true;
959 }
960 } catch (final Exception exception) {
961 respondOk(jinglePacket);
962 final Throwable rootCause = Throwables.getRootCause(exception);
963 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
964 // If this happens a termination is already in progress
965 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
966 return true;
967 }
968 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
969 webRTCWrapper.close();
970 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
971 return true;
972 }
973 }
974
975 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
976 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
977 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
978 throw new IllegalStateException("Invalid peer setup");
979 }
980 return peerSetup;
981 }
982
983 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
984 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
985 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
986 }
987 this.peerDtlsSetup = setup;
988 }
989
990 private boolean applyIceRestart(
991 final JinglePacket jinglePacket,
992 final RtpContentMap restartContentMap,
993 final boolean isOffer)
994 throws ExecutionException, InterruptedException {
995 final SessionDescription sessionDescription =
996 SessionDescription.of(restartContentMap, isResponder());
997 final org.webrtc.SessionDescription.Type type =
998 isOffer
999 ? org.webrtc.SessionDescription.Type.OFFER
1000 : org.webrtc.SessionDescription.Type.ANSWER;
1001 org.webrtc.SessionDescription sdp =
1002 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1003 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1004 if (isInitiator()) {
1005 // We ignore the offer and respond with tie-break. This will clause the responder
1006 // not to apply the content map
1007 return false;
1008 }
1009 }
1010 webRTCWrapper.setRemoteDescription(sdp).get();
1011 setRemoteContentMap(restartContentMap);
1012 if (isOffer) {
1013 final SessionDescription localSessionDescription = setLocalSessionDescription();
1014 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1015 // We need to respond OK before sending any candidates
1016 respondOk(jinglePacket);
1017 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1018 } else {
1019 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1020 }
1021 return true;
1022 }
1023
1024 private void processCandidates(
1025 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1026 contents) {
1027 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1028 content : contents) {
1029 processCandidate(content);
1030 }
1031 }
1032
1033 private void processCandidate(
1034 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1035 content) {
1036 final RtpContentMap rtpContentMap = getRemoteContentMap();
1037 final List<String> indices = toIdentificationTags(rtpContentMap);
1038 final String sdpMid = content.getKey(); // aka content name
1039 final IceUdpTransportInfo transport = content.getValue().transport;
1040 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1041
1042 // TODO check that credentials remained the same
1043
1044 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1045 final String sdp;
1046 try {
1047 sdp = candidate.toSdpAttribute(credentials.ufrag);
1048 } catch (final IllegalArgumentException e) {
1049 Log.d(
1050 Config.LOGTAG,
1051 id.account.getJid().asBareJid()
1052 + ": ignoring invalid ICE candidate "
1053 + e.getMessage());
1054 continue;
1055 }
1056 final int mLineIndex = indices.indexOf(sdpMid);
1057 if (mLineIndex < 0) {
1058 Log.w(
1059 Config.LOGTAG,
1060 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1061 }
1062 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1063 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1064 this.webRTCWrapper.addIceCandidate(iceCandidate);
1065 }
1066 }
1067
1068 private RtpContentMap getRemoteContentMap() {
1069 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1070 }
1071
1072 private RtpContentMap getLocalContentMap() {
1073 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1074 }
1075
1076 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1077 final Group originalGroup = rtpContentMap.group;
1078 final List<String> identificationTags =
1079 originalGroup == null
1080 ? rtpContentMap.getNames()
1081 : originalGroup.getIdentificationTags();
1082 if (identificationTags.size() == 0) {
1083 Log.w(
1084 Config.LOGTAG,
1085 id.account.getJid().asBareJid()
1086 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1087 }
1088 return identificationTags;
1089 }
1090
1091 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1092 final JinglePacket jinglePacket, final boolean expectVerification) {
1093 try {
1094 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1095 } catch (final Exception e) {
1096 return Futures.immediateFailedFuture(e);
1097 }
1098 }
1099
1100 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1101 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1102 Log.d(
1103 Config.LOGTAG,
1104 "receiveRtpContentMap("
1105 + receivedContentMap.getClass().getSimpleName()
1106 + ",expectVerification="
1107 + expectVerification
1108 + ")");
1109 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1110 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1111 id.account
1112 .getAxolotlService()
1113 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1114 return Futures.transform(
1115 future,
1116 omemoVerifiedPayload -> {
1117 // TODO test if an exception here triggers a correct abort
1118 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1119 Log.d(
1120 Config.LOGTAG,
1121 id.account.getJid().asBareJid()
1122 + ": received verifiable DTLS fingerprint via "
1123 + omemoVerification);
1124 return omemoVerifiedPayload.getPayload();
1125 },
1126 MoreExecutors.directExecutor());
1127 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1128 return Futures.immediateFailedFuture(
1129 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1130 } else {
1131 return Futures.immediateFuture(receivedContentMap);
1132 }
1133 }
1134
1135 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1136 if (isInitiator()) {
1137 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1138 return;
1139 }
1140 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1141 Futures.addCallback(
1142 future,
1143 new FutureCallback<>() {
1144 @Override
1145 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1146 receiveSessionInitiate(jinglePacket, rtpContentMap);
1147 }
1148
1149 @Override
1150 public void onFailure(@NonNull final Throwable throwable) {
1151 respondOk(jinglePacket);
1152 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1153 }
1154 },
1155 MoreExecutors.directExecutor());
1156 }
1157
1158 private void receiveSessionInitiate(
1159 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1160 try {
1161 contentMap.requireContentDescriptions();
1162 contentMap.requireDTLSFingerprint(true);
1163 } catch (final RuntimeException e) {
1164 Log.d(
1165 Config.LOGTAG,
1166 id.account.getJid().asBareJid() + ": improperly formatted contents",
1167 Throwables.getRootCause(e));
1168 respondOk(jinglePacket);
1169 sendSessionTerminate(Reason.of(e), e.getMessage());
1170 return;
1171 }
1172 Log.d(
1173 Config.LOGTAG,
1174 "processing session-init with " + contentMap.contents.size() + " contents");
1175 final State target;
1176 if (this.state == State.PROCEED) {
1177 Preconditions.checkState(
1178 proposedMedia != null && proposedMedia.size() > 0,
1179 "proposed media must be set when processing pre-approved session-initiate");
1180 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1181 sendSessionTerminate(
1182 Reason.SECURITY_ERROR,
1183 String.format(
1184 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1185 this.proposedMedia, contentMap.getMedia()));
1186 return;
1187 }
1188 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1189 } else {
1190 target = State.SESSION_INITIALIZED;
1191 setProposedMedia(contentMap.getMedia());
1192 }
1193 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1194 respondOk(jinglePacket);
1195 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1196 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1197 Log.d(
1198 Config.LOGTAG,
1199 id.account.getJid().asBareJid()
1200 + ": automatically accepting session-initiate");
1201 sendSessionAccept();
1202 } else {
1203 Log.d(
1204 Config.LOGTAG,
1205 id.account.getJid().asBareJid()
1206 + ": received not pre-approved session-initiate. start ringing");
1207 startRinging();
1208 }
1209 } else {
1210 Log.d(
1211 Config.LOGTAG,
1212 String.format(
1213 "%s: received session-initiate while in state %s",
1214 id.account.getJid().asBareJid(), state));
1215 terminateWithOutOfOrder(jinglePacket);
1216 }
1217 }
1218
1219 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1220 if (isResponder()) {
1221 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1222 return;
1223 }
1224 final ListenableFuture<RtpContentMap> future =
1225 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1226 Futures.addCallback(
1227 future,
1228 new FutureCallback<>() {
1229 @Override
1230 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1231 receiveSessionAccept(jinglePacket, rtpContentMap);
1232 }
1233
1234 @Override
1235 public void onFailure(@NonNull final Throwable throwable) {
1236 respondOk(jinglePacket);
1237 Log.d(
1238 Config.LOGTAG,
1239 id.account.getJid().asBareJid()
1240 + ": improperly formatted contents in session-accept",
1241 throwable);
1242 webRTCWrapper.close();
1243 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1244 }
1245 },
1246 MoreExecutors.directExecutor());
1247 }
1248
1249 private void receiveSessionAccept(
1250 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1251 try {
1252 contentMap.requireContentDescriptions();
1253 contentMap.requireDTLSFingerprint();
1254 } catch (final RuntimeException e) {
1255 respondOk(jinglePacket);
1256 Log.d(
1257 Config.LOGTAG,
1258 id.account.getJid().asBareJid()
1259 + ": improperly formatted contents in session-accept",
1260 e);
1261 webRTCWrapper.close();
1262 sendSessionTerminate(Reason.of(e), e.getMessage());
1263 return;
1264 }
1265 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1266 if (!initiatorMedia.equals(contentMap.getMedia())) {
1267 sendSessionTerminate(
1268 Reason.SECURITY_ERROR,
1269 String.format(
1270 "Your session-included included media %s but our session-initiate was %s",
1271 this.proposedMedia, contentMap.getMedia()));
1272 return;
1273 }
1274 Log.d(
1275 Config.LOGTAG,
1276 "processing session-accept with " + contentMap.contents.size() + " contents");
1277 if (transition(State.SESSION_ACCEPTED)) {
1278 respondOk(jinglePacket);
1279 receiveSessionAccept(contentMap);
1280 } else {
1281 Log.d(
1282 Config.LOGTAG,
1283 String.format(
1284 "%s: received session-accept while in state %s",
1285 id.account.getJid().asBareJid(), state));
1286 respondOk(jinglePacket);
1287 }
1288 }
1289
1290 private void receiveSessionAccept(final RtpContentMap contentMap) {
1291 this.responderRtpContentMap = contentMap;
1292 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1293 final SessionDescription sessionDescription;
1294 try {
1295 sessionDescription = SessionDescription.of(contentMap, false);
1296 } catch (final IllegalArgumentException | NullPointerException e) {
1297 Log.d(
1298 Config.LOGTAG,
1299 id.account.getJid().asBareJid()
1300 + ": unable convert offer from session-accept to SDP",
1301 e);
1302 webRTCWrapper.close();
1303 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1304 return;
1305 }
1306 final org.webrtc.SessionDescription answer =
1307 new org.webrtc.SessionDescription(
1308 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1309 try {
1310 this.webRTCWrapper.setRemoteDescription(answer).get();
1311 } catch (final Exception e) {
1312 Log.d(
1313 Config.LOGTAG,
1314 id.account.getJid().asBareJid()
1315 + ": unable to set remote description after receiving session-accept",
1316 Throwables.getRootCause(e));
1317 webRTCWrapper.close();
1318 sendSessionTerminate(
1319 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1320 return;
1321 }
1322 processCandidates(contentMap.contents.entrySet());
1323 }
1324
1325 private void sendSessionAccept() {
1326 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1327 if (rtpContentMap == null) {
1328 throw new IllegalStateException("initiator RTP Content Map has not been set");
1329 }
1330 final SessionDescription offer;
1331 try {
1332 offer = SessionDescription.of(rtpContentMap, true);
1333 } catch (final IllegalArgumentException | NullPointerException e) {
1334 Log.d(
1335 Config.LOGTAG,
1336 id.account.getJid().asBareJid()
1337 + ": unable convert offer from session-initiate to SDP",
1338 e);
1339 webRTCWrapper.close();
1340 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1341 return;
1342 }
1343 sendSessionAccept(rtpContentMap.getMedia(), offer);
1344 }
1345
1346 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1347 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1348 }
1349
1350 private synchronized void sendSessionAccept(
1351 final Set<Media> media,
1352 final SessionDescription offer,
1353 final List<PeerConnection.IceServer> iceServers) {
1354 if (isTerminated()) {
1355 Log.w(
1356 Config.LOGTAG,
1357 id.account.getJid().asBareJid()
1358 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1359 return;
1360 }
1361 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1362 try {
1363 setupWebRTC(media, iceServers, !includeCandidates);
1364 } catch (final WebRTCWrapper.InitializationException e) {
1365 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1366 webRTCWrapper.close();
1367 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1368 return;
1369 }
1370 final org.webrtc.SessionDescription sdp =
1371 new org.webrtc.SessionDescription(
1372 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1373 try {
1374 this.webRTCWrapper.setRemoteDescription(sdp).get();
1375 addIceCandidatesFromBlackLog();
1376 org.webrtc.SessionDescription webRTCSessionDescription =
1377 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1378 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1379 } catch (final Exception e) {
1380 failureToAcceptSession(e);
1381 }
1382 }
1383
1384 private void failureToAcceptSession(final Throwable throwable) {
1385 if (isTerminated()) {
1386 return;
1387 }
1388 final Throwable rootCause = Throwables.getRootCause(throwable);
1389 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1390 webRTCWrapper.close();
1391 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1392 }
1393
1394 private void failureToPerformAction(
1395 final JinglePacket.Action action, final Throwable throwable) {
1396 if (isTerminated()) {
1397 return;
1398 }
1399 final Throwable rootCause = Throwables.getRootCause(throwable);
1400 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1401 webRTCWrapper.close();
1402 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1403 }
1404
1405 private void addIceCandidatesFromBlackLog() {
1406 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1407 while ((foo = this.pendingIceCandidates.poll()) != null) {
1408 processCandidate(foo);
1409 Log.d(
1410 Config.LOGTAG,
1411 id.account.getJid().asBareJid() + ": added candidate from back log");
1412 }
1413 }
1414
1415 private void prepareSessionAccept(
1416 final org.webrtc.SessionDescription webRTCSessionDescription,
1417 final boolean includeCandidates) {
1418 final SessionDescription sessionDescription =
1419 SessionDescription.parse(webRTCSessionDescription.description);
1420 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1421 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1422 if (includeCandidates) {
1423 candidates = parseCandidates(sessionDescription);
1424 } else {
1425 candidates = ImmutableMultimap.of();
1426 }
1427 this.responderRtpContentMap = respondingRtpContentMap;
1428 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1429 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1430 prepareOutgoingContentMap(respondingRtpContentMap);
1431 Futures.addCallback(
1432 outgoingContentMapFuture,
1433 new FutureCallback<>() {
1434 @Override
1435 public void onSuccess(final RtpContentMap outgoingContentMap) {
1436 if (includeCandidates) {
1437 Log.d(
1438 Config.LOGTAG,
1439 "including "
1440 + candidates.size()
1441 + " candidates in session accept");
1442 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1443 } else {
1444 sendSessionAccept(outgoingContentMap);
1445 }
1446 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1447 }
1448
1449 @Override
1450 public void onFailure(@NonNull Throwable throwable) {
1451 failureToAcceptSession(throwable);
1452 }
1453 },
1454 MoreExecutors.directExecutor());
1455 }
1456
1457 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1458 if (isTerminated()) {
1459 Log.w(
1460 Config.LOGTAG,
1461 id.account.getJid().asBareJid()
1462 + ": preparing session accept was too slow. already terminated. nothing to do.");
1463 return;
1464 }
1465 transitionOrThrow(State.SESSION_ACCEPTED);
1466 final JinglePacket sessionAccept =
1467 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1468 send(sessionAccept);
1469 }
1470
1471 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1472 final RtpContentMap rtpContentMap) {
1473 if (this.omemoVerification.hasDeviceId()) {
1474 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1475 verifiedPayloadFuture =
1476 id.account
1477 .getAxolotlService()
1478 .encrypt(
1479 rtpContentMap,
1480 id.with,
1481 omemoVerification.getDeviceId());
1482 return Futures.transform(
1483 verifiedPayloadFuture,
1484 verifiedPayload -> {
1485 omemoVerification.setOrEnsureEqual(verifiedPayload);
1486 return verifiedPayload.getPayload();
1487 },
1488 MoreExecutors.directExecutor());
1489 } else {
1490 return Futures.immediateFuture(rtpContentMap);
1491 }
1492 }
1493
1494 synchronized void deliveryMessage(
1495 final Jid from,
1496 final Element message,
1497 final String serverMessageId,
1498 final long timestamp) {
1499 Log.d(
1500 Config.LOGTAG,
1501 id.account.getJid().asBareJid()
1502 + ": delivered message to JingleRtpConnection "
1503 + message);
1504 switch (message.getName()) {
1505 case "propose" -> receivePropose(
1506 from, Propose.upgrade(message), serverMessageId, timestamp);
1507 case "proceed" -> receiveProceed(
1508 from, Proceed.upgrade(message), serverMessageId, timestamp);
1509 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1510 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1511 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1512 }
1513 }
1514
1515 void deliverFailedProceed(final String message) {
1516 Log.d(
1517 Config.LOGTAG,
1518 id.account.getJid().asBareJid()
1519 + ": receive message error for proceed message ("
1520 + Strings.nullToEmpty(message)
1521 + ")");
1522 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1523 webRTCWrapper.close();
1524 Log.d(
1525 Config.LOGTAG,
1526 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1527 this.finish();
1528 }
1529 }
1530
1531 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1532 final boolean originatedFromMyself =
1533 from.asBareJid().equals(id.account.getJid().asBareJid());
1534 if (originatedFromMyself) {
1535 if (transition(State.ACCEPTED)) {
1536 acceptedOnOtherDevice(serverMsgId, timestamp);
1537 } else {
1538 Log.d(
1539 Config.LOGTAG,
1540 id.account.getJid().asBareJid()
1541 + ": unable to transition to accept because already in state="
1542 + this.state);
1543 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1544 }
1545 } else {
1546 Log.d(
1547 Config.LOGTAG,
1548 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1549 }
1550 }
1551
1552 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1553 if (serverMsgId != null) {
1554 this.message.setServerMsgId(serverMsgId);
1555 }
1556 this.message.setTime(timestamp);
1557 this.message.setCarbon(true); // indicate that call was accepted on other device
1558 this.writeLogMessageSuccess(0);
1559 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1560 this.finish();
1561 }
1562
1563 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1564 final boolean originatedFromMyself =
1565 from.asBareJid().equals(id.account.getJid().asBareJid());
1566 // reject from another one of my clients
1567 if (originatedFromMyself) {
1568 receiveRejectFromMyself(serverMsgId, timestamp);
1569 } else if (isInitiator()) {
1570 if (from.equals(id.with)) {
1571 receiveRejectFromResponder();
1572 } else {
1573 Log.d(
1574 Config.LOGTAG,
1575 id.account.getJid()
1576 + ": ignoring reject from "
1577 + from
1578 + " for session with "
1579 + id.with);
1580 }
1581 } else {
1582 Log.d(
1583 Config.LOGTAG,
1584 id.account.getJid()
1585 + ": ignoring reject from "
1586 + from
1587 + " for session with "
1588 + id.with);
1589 }
1590 }
1591
1592 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1593 if (transition(State.REJECTED)) {
1594 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1595 this.finish();
1596 if (serverMsgId != null) {
1597 this.message.setServerMsgId(serverMsgId);
1598 }
1599 this.message.setTime(timestamp);
1600 this.message.setCarbon(true); // indicate that call was rejected on other device
1601 writeLogMessageMissed();
1602 } else {
1603 Log.d(
1604 Config.LOGTAG,
1605 "not able to transition into REJECTED because already in " + this.state);
1606 }
1607 }
1608
1609 private void receiveRejectFromResponder() {
1610 if (isInState(State.PROCEED)) {
1611 Log.d(
1612 Config.LOGTAG,
1613 id.account.getJid()
1614 + ": received reject while still in proceed. callee reconsidered");
1615 closeTransitionLogFinish(State.REJECTED_RACED);
1616 return;
1617 }
1618 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1619 Log.d(
1620 Config.LOGTAG,
1621 id.account.getJid()
1622 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1623 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1624 return;
1625 }
1626 Log.d(
1627 Config.LOGTAG,
1628 id.account.getJid()
1629 + ": ignoring reject from responder because already in state "
1630 + this.state);
1631 }
1632
1633 private void receivePropose(
1634 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1635 final boolean originatedFromMyself =
1636 from.asBareJid().equals(id.account.getJid().asBareJid());
1637 if (originatedFromMyself) {
1638 Log.d(
1639 Config.LOGTAG,
1640 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1641 } else if (transition(
1642 State.PROPOSED,
1643 () -> {
1644 final Collection<RtpDescription> descriptions =
1645 Collections2.transform(
1646 Collections2.filter(
1647 propose.getDescriptions(),
1648 d -> d instanceof RtpDescription),
1649 input -> (RtpDescription) input);
1650 final Collection<Media> media =
1651 Collections2.transform(descriptions, RtpDescription::getMedia);
1652 Preconditions.checkState(
1653 !media.contains(Media.UNKNOWN),
1654 "RTP descriptions contain unknown media");
1655 Log.d(
1656 Config.LOGTAG,
1657 id.account.getJid().asBareJid()
1658 + ": received session proposal from "
1659 + from
1660 + " for "
1661 + media);
1662 this.setProposedMedia(Sets.newHashSet(media));
1663 })) {
1664 if (serverMsgId != null) {
1665 this.message.setServerMsgId(serverMsgId);
1666 }
1667 this.message.setTime(timestamp);
1668 startRinging();
1669 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1670 sendJingleMessage("ringing");
1671 }
1672 } else {
1673 Log.d(
1674 Config.LOGTAG,
1675 id.account.getJid()
1676 + ": ignoring session proposal because already in "
1677 + state);
1678 }
1679 }
1680
1681 private void startRinging() {
1682 this.callIntegration.setRinging();
1683 Log.d(
1684 Config.LOGTAG,
1685 id.account.getJid().asBareJid()
1686 + ": received call from "
1687 + id.with
1688 + ". start ringing");
1689 ringingTimeoutFuture =
1690 jingleConnectionManager.schedule(
1691 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1692 if (CallIntegration.selfManaged()) {
1693 return;
1694 }
1695 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1696 }
1697
1698 private synchronized void ringingTimeout() {
1699 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1700 switch (this.state) {
1701 case PROPOSED -> {
1702 message.markUnread();
1703 rejectCallFromProposed();
1704 }
1705 case SESSION_INITIALIZED -> {
1706 message.markUnread();
1707 rejectCallFromSessionInitiate();
1708 }
1709 }
1710 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1711 }
1712
1713 private void cancelRingingTimeout() {
1714 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1715 if (future != null && !future.isCancelled()) {
1716 future.cancel(false);
1717 }
1718 }
1719
1720 private void receiveProceed(
1721 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1722 final Set<Media> media =
1723 Preconditions.checkNotNull(
1724 this.proposedMedia, "Proposed media has to be set before handling proceed");
1725 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1726 if (from.equals(id.with)) {
1727 if (isInitiator()) {
1728 if (transition(State.PROCEED)) {
1729 if (serverMsgId != null) {
1730 this.message.setServerMsgId(serverMsgId);
1731 }
1732 this.message.setTime(timestamp);
1733 final Integer remoteDeviceId = proceed.getDeviceId();
1734 if (isOmemoEnabled()) {
1735 this.omemoVerification.setDeviceId(remoteDeviceId);
1736 } else {
1737 if (remoteDeviceId != null) {
1738 Log.d(
1739 Config.LOGTAG,
1740 id.account.getJid().asBareJid()
1741 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1742 }
1743 this.omemoVerification.setDeviceId(null);
1744 }
1745 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1746 } else {
1747 Log.d(
1748 Config.LOGTAG,
1749 String.format(
1750 "%s: ignoring proceed because already in %s",
1751 id.account.getJid().asBareJid(), this.state));
1752 }
1753 } else {
1754 Log.d(
1755 Config.LOGTAG,
1756 String.format(
1757 "%s: ignoring proceed because we were not initializing",
1758 id.account.getJid().asBareJid()));
1759 }
1760 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1761 if (transition(State.ACCEPTED)) {
1762 Log.d(
1763 Config.LOGTAG,
1764 id.account.getJid().asBareJid()
1765 + ": moved session with "
1766 + id.with
1767 + " into state accepted after received carbon copied proceed");
1768 acceptedOnOtherDevice(serverMsgId, timestamp);
1769 }
1770 } else {
1771 Log.d(
1772 Config.LOGTAG,
1773 String.format(
1774 "%s: ignoring proceed from %s. was expected from %s",
1775 id.account.getJid().asBareJid(), from, id.with));
1776 }
1777 }
1778
1779 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1780 if (from.equals(id.with)) {
1781 final State target =
1782 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1783 if (transition(target)) {
1784 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1785 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1786 Log.d(
1787 Config.LOGTAG,
1788 id.account.getJid().asBareJid()
1789 + ": session with "
1790 + id.with
1791 + " has been retracted (serverMsgId="
1792 + serverMsgId
1793 + ")");
1794 if (serverMsgId != null) {
1795 this.message.setServerMsgId(serverMsgId);
1796 }
1797 this.message.setTime(timestamp);
1798 if (target == State.RETRACTED) {
1799 this.message.markUnread();
1800 }
1801 writeLogMessageMissed();
1802 finish();
1803 } else {
1804 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1805 }
1806 } else {
1807 // TODO parse retract from self
1808 Log.d(
1809 Config.LOGTAG,
1810 id.account.getJid().asBareJid()
1811 + ": received retract from "
1812 + from
1813 + ". expected retract from"
1814 + id.with
1815 + ". ignoring");
1816 }
1817 }
1818
1819 public void sendSessionInitiate() {
1820 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1821 }
1822
1823 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1824 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1825 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1826 }
1827
1828 private synchronized void sendSessionInitiate(
1829 final Set<Media> media,
1830 final State targetState,
1831 final List<PeerConnection.IceServer> iceServers) {
1832 if (isTerminated()) {
1833 Log.w(
1834 Config.LOGTAG,
1835 id.account.getJid().asBareJid()
1836 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1837 return;
1838 }
1839 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1840 try {
1841 setupWebRTC(media, iceServers, !includeCandidates);
1842 } catch (final WebRTCWrapper.InitializationException e) {
1843 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1844 webRTCWrapper.close();
1845 sendRetract(Reason.ofThrowable(e));
1846 return;
1847 }
1848 try {
1849 org.webrtc.SessionDescription webRTCSessionDescription =
1850 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1851 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1852 } catch (final Exception e) {
1853 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1854 // exceptions
1855 failureToInitiateSession(e, targetState);
1856 }
1857 }
1858
1859 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1860 if (isTerminated()) {
1861 return;
1862 }
1863 Log.d(
1864 Config.LOGTAG,
1865 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1866 Throwables.getRootCause(throwable));
1867 webRTCWrapper.close();
1868 final Reason reason = Reason.ofThrowable(throwable);
1869 if (isInState(targetState)) {
1870 sendSessionTerminate(reason, throwable.getMessage());
1871 } else {
1872 sendRetract(reason);
1873 }
1874 }
1875
1876 private void sendRetract(final Reason reason) {
1877 // TODO embed reason into retract
1878 sendJingleMessage("retract", id.with.asBareJid());
1879 transitionOrThrow(reasonToState(reason));
1880 this.finish();
1881 }
1882
1883 private void prepareSessionInitiate(
1884 final org.webrtc.SessionDescription webRTCSessionDescription,
1885 final boolean includeCandidates,
1886 final State targetState) {
1887 final SessionDescription sessionDescription =
1888 SessionDescription.parse(webRTCSessionDescription.description);
1889 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1890 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1891 if (includeCandidates) {
1892 candidates = parseCandidates(sessionDescription);
1893 } else {
1894 candidates = ImmutableMultimap.of();
1895 }
1896 this.initiatorRtpContentMap = rtpContentMap;
1897 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1898 encryptSessionInitiate(rtpContentMap);
1899 Futures.addCallback(
1900 outgoingContentMapFuture,
1901 new FutureCallback<>() {
1902 @Override
1903 public void onSuccess(final RtpContentMap outgoingContentMap) {
1904 if (includeCandidates) {
1905 Log.d(
1906 Config.LOGTAG,
1907 "including "
1908 + candidates.size()
1909 + " candidates in session initiate");
1910 sendSessionInitiate(
1911 outgoingContentMap.withCandidates(candidates), targetState);
1912 } else {
1913 sendSessionInitiate(outgoingContentMap, targetState);
1914 }
1915 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1916 }
1917
1918 @Override
1919 public void onFailure(@NonNull final Throwable throwable) {
1920 failureToInitiateSession(throwable, targetState);
1921 }
1922 },
1923 MoreExecutors.directExecutor());
1924 }
1925
1926 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1927 if (isTerminated()) {
1928 Log.w(
1929 Config.LOGTAG,
1930 id.account.getJid().asBareJid()
1931 + ": preparing session was too slow. already terminated. nothing to do.");
1932 return;
1933 }
1934 this.transitionOrThrow(targetState);
1935 final JinglePacket sessionInitiate =
1936 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1937 send(sessionInitiate);
1938 }
1939
1940 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1941 final RtpContentMap rtpContentMap) {
1942 if (this.omemoVerification.hasDeviceId()) {
1943 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1944 verifiedPayloadFuture =
1945 id.account
1946 .getAxolotlService()
1947 .encrypt(
1948 rtpContentMap,
1949 id.with,
1950 omemoVerification.getDeviceId());
1951 final ListenableFuture<RtpContentMap> future =
1952 Futures.transform(
1953 verifiedPayloadFuture,
1954 verifiedPayload -> {
1955 omemoVerification.setSessionFingerprint(
1956 verifiedPayload.getFingerprint());
1957 return verifiedPayload.getPayload();
1958 },
1959 MoreExecutors.directExecutor());
1960 if (Config.REQUIRE_RTP_VERIFICATION) {
1961 return future;
1962 }
1963 return Futures.catching(
1964 future,
1965 CryptoFailedException.class,
1966 e -> {
1967 Log.w(
1968 Config.LOGTAG,
1969 id.account.getJid().asBareJid()
1970 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1971 e);
1972 return rtpContentMap;
1973 },
1974 MoreExecutors.directExecutor());
1975 } else {
1976 return Futures.immediateFuture(rtpContentMap);
1977 }
1978 }
1979
1980 protected void sendSessionTerminate(final Reason reason) {
1981 sendSessionTerminate(reason, null);
1982 }
1983
1984 protected void sendSessionTerminate(final Reason reason, final String text) {
1985 sendSessionTerminate(reason, text, this::writeLogMessage);
1986 }
1987
1988 private void sendTransportInfo(
1989 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1990 final RtpContentMap transportInfo;
1991 try {
1992 final RtpContentMap rtpContentMap =
1993 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1994 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1995 } catch (final Exception e) {
1996 Log.d(
1997 Config.LOGTAG,
1998 id.account.getJid().asBareJid()
1999 + ": unable to prepare transport-info from candidate for content="
2000 + contentName);
2001 return;
2002 }
2003 final JinglePacket jinglePacket =
2004 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2005 send(jinglePacket);
2006 }
2007
2008 public RtpEndUserState getEndUserState() {
2009 switch (this.state) {
2010 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2011 if (isInitiator()) {
2012 return RtpEndUserState.RINGING;
2013 } else {
2014 return RtpEndUserState.INCOMING_CALL;
2015 }
2016 }
2017 case PROCEED -> {
2018 if (isInitiator()) {
2019 return RtpEndUserState.RINGING;
2020 } else {
2021 return RtpEndUserState.ACCEPTING_CALL;
2022 }
2023 }
2024 case SESSION_INITIALIZED_PRE_APPROVED -> {
2025 if (isInitiator()) {
2026 return RtpEndUserState.RINGING;
2027 } else {
2028 return RtpEndUserState.CONNECTING;
2029 }
2030 }
2031 case SESSION_ACCEPTED -> {
2032 final ContentAddition ca = getPendingContentAddition();
2033 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2034 return RtpEndUserState.INCOMING_CONTENT_ADD;
2035 }
2036 return getPeerConnectionStateAsEndUserState();
2037 }
2038 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2039 if (isInitiator()) {
2040 return RtpEndUserState.DECLINED_OR_BUSY;
2041 } else {
2042 return RtpEndUserState.ENDED;
2043 }
2044 }
2045 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2046 return RtpEndUserState.ENDED;
2047 }
2048 case RETRACTED_RACED -> {
2049 if (isInitiator()) {
2050 return RtpEndUserState.ENDED;
2051 } else {
2052 return RtpEndUserState.RETRACTED;
2053 }
2054 }
2055 case TERMINATED_CONNECTIVITY_ERROR -> {
2056 return zeroDuration()
2057 ? RtpEndUserState.CONNECTIVITY_ERROR
2058 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2059 }
2060 case TERMINATED_APPLICATION_FAILURE -> {
2061 return RtpEndUserState.APPLICATION_ERROR;
2062 }
2063 case TERMINATED_SECURITY_ERROR -> {
2064 return RtpEndUserState.SECURITY_ERROR;
2065 }
2066 }
2067 throw new IllegalStateException(
2068 String.format("%s has no equivalent EndUserState", this.state));
2069 }
2070
2071 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2072 final PeerConnection.PeerConnectionState state;
2073 try {
2074 state = webRTCWrapper.getState();
2075 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2076 // We usually close the WebRTCWrapper *before* transitioning so we might still
2077 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2078 return RtpEndUserState.ENDING_CALL;
2079 }
2080 return switch (state) {
2081 case CONNECTED -> RtpEndUserState.CONNECTED;
2082 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2083 case CLOSED -> RtpEndUserState.ENDING_CALL;
2084 default -> zeroDuration()
2085 ? RtpEndUserState.CONNECTIVITY_ERROR
2086 : RtpEndUserState.RECONNECTING;
2087 };
2088 }
2089
2090 private boolean isPeerConnectionConnected() {
2091 try {
2092 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2093 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2094 return false;
2095 }
2096 }
2097
2098 private void updateCallIntegrationState() {
2099 switch (this.state) {
2100 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2101 if (isInitiator()) {
2102 this.callIntegration.setDialing();
2103 } else {
2104 this.callIntegration.setRinging();
2105 }
2106 }
2107 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2108 if (isInitiator()) {
2109 this.callIntegration.setDialing();
2110 } else {
2111 this.callIntegration.setInitialized();
2112 }
2113 }
2114 case SESSION_ACCEPTED -> {
2115 if (isPeerConnectionConnected()) {
2116 this.callIntegration.setActive();
2117 } else {
2118 this.callIntegration.setInitialized();
2119 }
2120 }
2121 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2122 if (isInitiator()) {
2123 this.callIntegration.busy();
2124 } else {
2125 this.callIntegration.rejected();
2126 }
2127 }
2128 case TERMINATED_SUCCESS -> this.callIntegration.success();
2129 case ACCEPTED -> this.callIntegration.accepted();
2130 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2131 .retracted();
2132 case TERMINATED_CONNECTIVITY_ERROR,
2133 TERMINATED_APPLICATION_FAILURE,
2134 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2135 default -> throw new IllegalStateException(
2136 String.format("%s is not handled", this.state));
2137 }
2138 }
2139
2140 public ContentAddition getPendingContentAddition() {
2141 final RtpContentMap in = this.incomingContentAdd;
2142 final RtpContentMap out = this.outgoingContentAdd;
2143 if (out != null) {
2144 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2145 } else if (in != null) {
2146 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2147 } else {
2148 return null;
2149 }
2150 }
2151
2152 public Set<Media> getMedia() {
2153 final State current = getState();
2154 if (current == State.NULL) {
2155 if (isInitiator()) {
2156 return Preconditions.checkNotNull(
2157 this.proposedMedia, "RTP connection has not been initialized properly");
2158 }
2159 throw new IllegalStateException("RTP connection has not been initialized yet");
2160 }
2161 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2162 return Preconditions.checkNotNull(
2163 this.proposedMedia, "RTP connection has not been initialized properly");
2164 }
2165 final RtpContentMap localContentMap = getLocalContentMap();
2166 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2167 if (localContentMap != null) {
2168 return localContentMap.getMedia();
2169 } else if (initiatorContentMap != null) {
2170 return initiatorContentMap.getMedia();
2171 } else if (isTerminated()) {
2172 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2173 } else {
2174 return Preconditions.checkNotNull(
2175 this.proposedMedia, "RTP connection has not been initialized properly");
2176 }
2177 }
2178
2179 public boolean isVerified() {
2180 final String fingerprint = this.omemoVerification.getFingerprint();
2181 if (fingerprint == null) {
2182 return false;
2183 }
2184 final FingerprintStatus status =
2185 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2186 return status != null && status.isVerified();
2187 }
2188
2189 public boolean addMedia(final Media media) {
2190 final Set<Media> currentMedia = getMedia();
2191 if (currentMedia.contains(media)) {
2192 throw new IllegalStateException(String.format("%s has already been proposed", media));
2193 }
2194 // TODO add state protection - can only add while ACCEPTED or so
2195 Log.d(Config.LOGTAG, "adding media: " + media);
2196 return webRTCWrapper.addTrack(media);
2197 }
2198
2199 public synchronized void acceptCall() {
2200 switch (this.state) {
2201 case PROPOSED -> {
2202 cancelRingingTimeout();
2203 acceptCallFromProposed();
2204 }
2205 case SESSION_INITIALIZED -> {
2206 cancelRingingTimeout();
2207 acceptCallFromSessionInitialized();
2208 }
2209 case ACCEPTED -> Log.w(
2210 Config.LOGTAG,
2211 id.account.getJid().asBareJid()
2212 + ": the call has already been accepted with another client. UI was just lagging behind");
2213 case PROCEED, SESSION_ACCEPTED -> Log.w(
2214 Config.LOGTAG,
2215 id.account.getJid().asBareJid()
2216 + ": the call has already been accepted. user probably double tapped the UI");
2217 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2218 }
2219 }
2220
2221 public synchronized void rejectCall() {
2222 if (isTerminated()) {
2223 Log.w(
2224 Config.LOGTAG,
2225 id.account.getJid().asBareJid()
2226 + ": received rejectCall() when session has already been terminated. nothing to do");
2227 return;
2228 }
2229 switch (this.state) {
2230 case PROPOSED -> rejectCallFromProposed();
2231 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2232 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2233 }
2234 }
2235
2236 public synchronized void endCall() {
2237 if (isTerminated()) {
2238 Log.w(
2239 Config.LOGTAG,
2240 id.account.getJid().asBareJid()
2241 + ": received endCall() when session has already been terminated. nothing to do");
2242 return;
2243 }
2244 if (isInState(State.PROPOSED) && isResponder()) {
2245 rejectCallFromProposed();
2246 return;
2247 }
2248 if (isInState(State.PROCEED)) {
2249 if (isInitiator()) {
2250 retractFromProceed();
2251 } else {
2252 rejectCallFromProceed();
2253 }
2254 return;
2255 }
2256 if (isInitiator()
2257 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2258 this.webRTCWrapper.close();
2259 sendSessionTerminate(Reason.CANCEL);
2260 return;
2261 }
2262 if (isInState(State.SESSION_INITIALIZED)) {
2263 rejectCallFromSessionInitiate();
2264 return;
2265 }
2266 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2267 this.webRTCWrapper.close();
2268 sendSessionTerminate(Reason.SUCCESS);
2269 return;
2270 }
2271 if (isInState(
2272 State.TERMINATED_APPLICATION_FAILURE,
2273 State.TERMINATED_CONNECTIVITY_ERROR,
2274 State.TERMINATED_DECLINED_OR_BUSY)) {
2275 Log.d(
2276 Config.LOGTAG,
2277 "ignoring request to end call because already in state " + this.state);
2278 return;
2279 }
2280 throw new IllegalStateException(
2281 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2282 }
2283
2284 private void retractFromProceed() {
2285 Log.d(Config.LOGTAG, "retract from proceed");
2286 this.sendJingleMessage("retract");
2287 closeTransitionLogFinish(State.RETRACTED_RACED);
2288 }
2289
2290 private void closeTransitionLogFinish(final State state) {
2291 this.webRTCWrapper.close();
2292 transitionOrThrow(state);
2293 writeLogMessage(state);
2294 finish();
2295 }
2296
2297 private void setupWebRTC(
2298 final Set<Media> media,
2299 final List<PeerConnection.IceServer> iceServers,
2300 final boolean trickle)
2301 throws WebRTCWrapper.InitializationException {
2302 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2303 this.webRTCWrapper.setup(
2304 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2305 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2306 }
2307
2308 private void acceptCallFromProposed() {
2309 transitionOrThrow(State.PROCEED);
2310 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2311 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2312 this.sendJingleMessage("proceed");
2313 }
2314
2315 private void rejectCallFromProposed() {
2316 transitionOrThrow(State.REJECTED);
2317 writeLogMessageMissed();
2318 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2319 this.sendJingleMessage("reject");
2320 finish();
2321 }
2322
2323 private void rejectCallFromProceed() {
2324 this.sendJingleMessage("reject");
2325 closeTransitionLogFinish(State.REJECTED_RACED);
2326 }
2327
2328 private void rejectCallFromSessionInitiate() {
2329 webRTCWrapper.close();
2330 sendSessionTerminate(Reason.DECLINE);
2331 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2332 }
2333
2334 private void sendJingleMessage(final String action) {
2335 sendJingleMessage(action, id.with);
2336 }
2337
2338 private void sendJingleMessage(final String action, final Jid to) {
2339 final MessagePacket messagePacket = new MessagePacket();
2340 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2341 messagePacket.setTo(to);
2342 final Element intent =
2343 messagePacket
2344 .addChild(action, Namespace.JINGLE_MESSAGE)
2345 .setAttribute("id", id.sessionId);
2346 if ("proceed".equals(action)) {
2347 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2348 if (isOmemoEnabled()) {
2349 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2350 final Element device =
2351 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2352 device.setAttribute("id", deviceId);
2353 }
2354 }
2355 messagePacket.addChild("store", "urn:xmpp:hints");
2356 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2357 }
2358
2359 private boolean isOmemoEnabled() {
2360 final Conversational conversational = message.getConversation();
2361 if (conversational instanceof Conversation) {
2362 return ((Conversation) conversational).getNextEncryption()
2363 == Message.ENCRYPTION_AXOLOTL;
2364 }
2365 return false;
2366 }
2367
2368 private void acceptCallFromSessionInitialized() {
2369 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2370 sendSessionAccept();
2371 }
2372
2373 @Override
2374 protected synchronized boolean transition(final State target, final Runnable runnable) {
2375 if (super.transition(target, runnable)) {
2376 updateEndUserState();
2377 updateOngoingCallNotification();
2378 return true;
2379 } else {
2380 return false;
2381 }
2382 }
2383
2384 @Override
2385 public void onIceCandidate(final IceCandidate iceCandidate) {
2386 final RtpContentMap rtpContentMap =
2387 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2388 final IceUdpTransportInfo.Credentials credentials;
2389 try {
2390 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2391 } catch (final IllegalArgumentException e) {
2392 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2393 return;
2394 }
2395 final String uFrag = credentials.ufrag;
2396 final IceUdpTransportInfo.Candidate candidate =
2397 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2398 if (candidate == null) {
2399 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2400 return;
2401 }
2402 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2403 sendTransportInfo(iceCandidate.sdpMid, candidate);
2404 }
2405
2406 @Override
2407 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2408 Log.d(
2409 Config.LOGTAG,
2410 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2411 this.stateHistory.add(newState);
2412 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2413 this.sessionDuration.start();
2414 updateOngoingCallNotification();
2415 } else if (this.sessionDuration.isRunning()) {
2416 this.sessionDuration.stop();
2417 updateOngoingCallNotification();
2418 }
2419
2420 final boolean neverConnected =
2421 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2422
2423 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2424 if (neverConnected) {
2425 if (isTerminated()) {
2426 Log.d(
2427 Config.LOGTAG,
2428 id.account.getJid().asBareJid()
2429 + ": not sending session-terminate after connectivity error because session is already in state "
2430 + this.state);
2431 return;
2432 }
2433 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2434 return;
2435 } else {
2436 this.restartIce();
2437 }
2438 }
2439 updateEndUserState();
2440 }
2441
2442 private void restartIce() {
2443 this.stateHistory.clear();
2444 this.webRTCWrapper.restartIceAsync();
2445 }
2446
2447 @Override
2448 public void onRenegotiationNeeded() {
2449 this.webRTCWrapper.execute(this::renegotiate);
2450 }
2451
2452 private void renegotiate() {
2453 final SessionDescription sessionDescription;
2454 try {
2455 sessionDescription = setLocalSessionDescription();
2456 } catch (final Exception e) {
2457 final Throwable cause = Throwables.getRootCause(e);
2458 webRTCWrapper.close();
2459 if (isTerminated()) {
2460 Log.d(
2461 Config.LOGTAG,
2462 "failed to renegotiate. session was already terminated",
2463 cause);
2464 return;
2465 }
2466 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2467 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2468 return;
2469 }
2470 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2471 final RtpContentMap currentContentMap = getLocalContentMap();
2472 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2473 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2474
2475 Log.d(
2476 Config.LOGTAG,
2477 id.getAccount().getJid().asBareJid()
2478 + ": renegotiate. iceRestart="
2479 + iceRestart
2480 + " content id diff="
2481 + diff);
2482
2483 if (diff.hasModifications() && iceRestart) {
2484 webRTCWrapper.close();
2485 sendSessionTerminate(
2486 Reason.FAILED_APPLICATION,
2487 "WebRTC unexpectedly tried to modify content and transport at once");
2488 return;
2489 }
2490
2491 if (iceRestart) {
2492 initiateIceRestart(rtpContentMap);
2493 return;
2494 } else if (diff.isEmpty()) {
2495 Log.d(
2496 Config.LOGTAG,
2497 "renegotiation. nothing to do. SignalingState="
2498 + this.webRTCWrapper.getSignalingState());
2499 }
2500
2501 if (diff.added.size() > 0) {
2502 modifyLocalContentMap(rtpContentMap);
2503 sendContentAdd(rtpContentMap, diff.added);
2504 }
2505 }
2506
2507 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2508 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2509 final JinglePacket jinglePacket =
2510 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2511 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2512 jinglePacket.setTo(id.with);
2513 xmppConnectionService.sendIqPacket(
2514 id.account,
2515 jinglePacket,
2516 (account, response) -> {
2517 if (response.getType() == IqPacket.TYPE.RESULT) {
2518 Log.d(Config.LOGTAG, "received success to our ice restart");
2519 setLocalContentMap(rtpContentMap);
2520 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2521 return;
2522 }
2523 if (response.getType() == IqPacket.TYPE.ERROR) {
2524 if (isTieBreak(response)) {
2525 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2526 return;
2527 }
2528 handleIqErrorResponse(response);
2529 }
2530 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2531 handleIqTimeoutResponse(response);
2532 }
2533 });
2534 }
2535
2536 private boolean isTieBreak(final IqPacket response) {
2537 final Element error = response.findChild("error");
2538 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2539 }
2540
2541 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2542 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2543 this.outgoingContentAdd = contentAdd;
2544 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2545 prepareOutgoingContentMap(contentAdd);
2546 Futures.addCallback(
2547 outgoingContentMapFuture,
2548 new FutureCallback<>() {
2549 @Override
2550 public void onSuccess(final RtpContentMap outgoingContentMap) {
2551 sendContentAdd(outgoingContentMap);
2552 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2553 }
2554
2555 @Override
2556 public void onFailure(@NonNull Throwable throwable) {
2557 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2558 }
2559 },
2560 MoreExecutors.directExecutor());
2561 }
2562
2563 private void sendContentAdd(final RtpContentMap contentAdd) {
2564
2565 final JinglePacket jinglePacket =
2566 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2567 jinglePacket.setTo(id.with);
2568 xmppConnectionService.sendIqPacket(
2569 id.account,
2570 jinglePacket,
2571 (connection, response) -> {
2572 if (response.getType() == IqPacket.TYPE.RESULT) {
2573 Log.d(
2574 Config.LOGTAG,
2575 id.getAccount().getJid().asBareJid()
2576 + ": received ACK to our content-add");
2577 return;
2578 }
2579 if (response.getType() == IqPacket.TYPE.ERROR) {
2580 if (isTieBreak(response)) {
2581 this.outgoingContentAdd = null;
2582 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2583 return;
2584 }
2585 handleIqErrorResponse(response);
2586 }
2587 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2588 handleIqTimeoutResponse(response);
2589 }
2590 });
2591 }
2592
2593 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2594 if (isInitiator()) {
2595 this.initiatorRtpContentMap = rtpContentMap;
2596 } else {
2597 this.responderRtpContentMap = rtpContentMap;
2598 }
2599 }
2600
2601 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2602 if (isInitiator()) {
2603 this.responderRtpContentMap = rtpContentMap;
2604 } else {
2605 this.initiatorRtpContentMap = rtpContentMap;
2606 }
2607 }
2608
2609 // this method is to be used for content map modifications that modify media
2610 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2611 final RtpContentMap activeContents = rtpContentMap.activeContents();
2612 setLocalContentMap(activeContents);
2613 this.callIntegration.setAudioDeviceWhenAvailable(
2614 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2615 updateEndUserState();
2616 }
2617
2618 private SessionDescription setLocalSessionDescription()
2619 throws ExecutionException, InterruptedException {
2620 final org.webrtc.SessionDescription sessionDescription =
2621 this.webRTCWrapper.setLocalDescription(false).get();
2622 return SessionDescription.parse(sessionDescription.description);
2623 }
2624
2625 private void closeWebRTCSessionAfterFailedConnection() {
2626 this.webRTCWrapper.close();
2627 synchronized (this) {
2628 if (isTerminated()) {
2629 Log.d(
2630 Config.LOGTAG,
2631 id.account.getJid().asBareJid()
2632 + ": no need to send session-terminate after failed connection. Other party already did");
2633 return;
2634 }
2635 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2636 }
2637 }
2638
2639 public boolean zeroDuration() {
2640 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2641 }
2642
2643 public long getCallDuration() {
2644 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2645 }
2646
2647 public CallIntegration getCallIntegration() {
2648 return this.callIntegration;
2649 }
2650
2651 public boolean isMicrophoneEnabled() {
2652 return webRTCWrapper.isMicrophoneEnabled();
2653 }
2654
2655 public boolean setMicrophoneEnabled(final boolean enabled) {
2656 return webRTCWrapper.setMicrophoneEnabled(enabled);
2657 }
2658
2659 public boolean isVideoEnabled() {
2660 return webRTCWrapper.isVideoEnabled();
2661 }
2662
2663 public void setVideoEnabled(final boolean enabled) {
2664 webRTCWrapper.setVideoEnabled(enabled);
2665 }
2666
2667 public boolean isCameraSwitchable() {
2668 return webRTCWrapper.isCameraSwitchable();
2669 }
2670
2671 public boolean isFrontCamera() {
2672 return webRTCWrapper.isFrontCamera();
2673 }
2674
2675 public ListenableFuture<Boolean> switchCamera() {
2676 return webRTCWrapper.switchCamera();
2677 }
2678
2679 @Override
2680 public void onCallIntegrationShowIncomingCallUi() {
2681 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2682 }
2683
2684 @Override
2685 public void onCallIntegrationDisconnect() {
2686 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2687 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2688 rejectCall();
2689 } else {
2690 endCall();
2691 }
2692 }
2693
2694 @Override
2695 public void onCallIntegrationReject() {
2696 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2697 try {
2698 rejectCall();
2699 } catch (final IllegalStateException e) {
2700 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2701 }
2702 }
2703
2704 @Override
2705 public void onCallIntegrationAnswer() {
2706 // we need to start the UI to a) show it and b) be able to ask for permissions
2707 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2708 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2709 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2710 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2711 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2712 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2713 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2714 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2715 xmppConnectionService.startActivity(intent);
2716 }
2717
2718 @Override
2719 public void onAudioDeviceChanged(
2720 final CallIntegration.AudioDevice selectedAudioDevice,
2721 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2722 Log.d(
2723 Config.LOGTAG,
2724 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2725 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2726 selectedAudioDevice, availableAudioDevices);
2727 }
2728
2729 private void updateEndUserState() {
2730 final RtpEndUserState endUserState = getEndUserState();
2731 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2732 this.updateCallIntegrationState();
2733 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2734 id.account, id.with, id.sessionId, endUserState);
2735 }
2736
2737 private void updateOngoingCallNotification() {
2738 final State state = this.state;
2739 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2740 final boolean reconnecting;
2741 if (state == State.SESSION_ACCEPTED) {
2742 reconnecting =
2743 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2744 } else {
2745 reconnecting = false;
2746 }
2747 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2748 } else {
2749 xmppConnectionService.removeOngoingCall();
2750 }
2751 }
2752
2753 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2754 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2755 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2756 request.setTo(id.account.getDomain());
2757 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2758 xmppConnectionService.sendIqPacket(
2759 id.account,
2760 request,
2761 (account, response) -> {
2762 final var iceServers = IceServers.parse(response);
2763 if (iceServers.size() == 0) {
2764 Log.w(
2765 Config.LOGTAG,
2766 id.account.getJid().asBareJid()
2767 + ": no ICE server found "
2768 + response);
2769 }
2770 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2771 });
2772 } else {
2773 Log.w(
2774 Config.LOGTAG,
2775 id.account.getJid().asBareJid() + ": has no external service discovery");
2776 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2777 }
2778 }
2779
2780 @Override
2781 protected void terminateTransport() {
2782 this.webRTCWrapper.close();
2783 }
2784
2785 @Override
2786 protected void finish() {
2787 if (isTerminated()) {
2788 this.cancelRingingTimeout();
2789 this.callIntegration.verifyDisconnected();
2790 this.webRTCWrapper.verifyClosed();
2791 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2792 super.finish();
2793 } else {
2794 throw new IllegalStateException(
2795 String.format("Unable to call finish from %s", this.state));
2796 }
2797 }
2798
2799 private void writeLogMessage(final State state) {
2800 final long duration = getCallDuration();
2801 if (state == State.TERMINATED_SUCCESS
2802 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2803 writeLogMessageSuccess(duration);
2804 } else {
2805 writeLogMessageMissed();
2806 }
2807 }
2808
2809 private void writeLogMessageSuccess(final long duration) {
2810 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2811 this.writeMessage();
2812 }
2813
2814 private void writeLogMessageMissed() {
2815 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2816 this.writeMessage();
2817 }
2818
2819 private void writeMessage() {
2820 final Conversational conversational = message.getConversation();
2821 if (conversational instanceof Conversation) {
2822 ((Conversation) conversational).add(this.message);
2823 xmppConnectionService.createMessageAsync(message);
2824 xmppConnectionService.updateConversationUi();
2825 } else {
2826 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2827 }
2828 }
2829
2830 public Optional<VideoTrack> getLocalVideoTrack() {
2831 return webRTCWrapper.getLocalVideoTrack();
2832 }
2833
2834 public Optional<VideoTrack> getRemoteVideoTrack() {
2835 return webRTCWrapper.getRemoteVideoTrack();
2836 }
2837
2838 public EglBase.Context getEglBaseContext() {
2839 return webRTCWrapper.getEglBaseContext();
2840 }
2841
2842 void setProposedMedia(final Set<Media> media) {
2843 this.proposedMedia = media;
2844 this.callIntegration.setVideoState(
2845 Media.audioOnly(media)
2846 ? VideoProfile.STATE_AUDIO_ONLY
2847 : VideoProfile.STATE_BIDIRECTIONAL);
2848 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2849 }
2850
2851 public void fireStateUpdate() {
2852 final RtpEndUserState endUserState = getEndUserState();
2853 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2854 id.account, id.with, id.sessionId, endUserState);
2855 }
2856
2857 public boolean isSwitchToVideoAvailable() {
2858 final boolean prerequisite =
2859 Media.audioOnly(getMedia())
2860 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2861 .contains(getEndUserState());
2862 return prerequisite && remoteHasVideoFeature();
2863 }
2864
2865 private boolean remoteHasVideoFeature() {
2866 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2867 }
2868
2869 private boolean remoteHasSdpOfferAnswer() {
2870 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2871 }
2872
2873 private interface OnIceServersDiscovered {
2874 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2875 }
2876}