1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.util.Log;
6
7import androidx.annotation.NonNull;
8import androidx.annotation.Nullable;
9
10import com.google.common.base.Joiner;
11import com.google.common.base.Optional;
12import com.google.common.base.Preconditions;
13import com.google.common.base.Stopwatch;
14import com.google.common.base.Strings;
15import com.google.common.base.Throwables;
16import com.google.common.collect.Collections2;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Sets;
22import com.google.common.util.concurrent.FutureCallback;
23import com.google.common.util.concurrent.Futures;
24import com.google.common.util.concurrent.ListenableFuture;
25import com.google.common.util.concurrent.MoreExecutors;
26
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Conversation;
33import eu.siacs.conversations.entities.Conversational;
34import eu.siacs.conversations.entities.Message;
35import eu.siacs.conversations.entities.RtpSessionStatus;
36import eu.siacs.conversations.services.AppRTCAudioManager;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.ui.RtpSessionActivity;
39import eu.siacs.conversations.xml.Element;
40import eu.siacs.conversations.xml.Namespace;
41import eu.siacs.conversations.xmpp.Jid;
42import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
44import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
45import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import eu.siacs.conversations.xmpp.stanzas.IqPacket;
51import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
52
53import org.webrtc.EglBase;
54import org.webrtc.IceCandidate;
55import org.webrtc.PeerConnection;
56import org.webrtc.VideoTrack;
57
58import java.util.Arrays;
59import java.util.Collection;
60import java.util.Collections;
61import java.util.LinkedList;
62import java.util.List;
63import java.util.Map;
64import java.util.Queue;
65import java.util.Set;
66import java.util.concurrent.ExecutionException;
67import java.util.concurrent.ScheduledFuture;
68import java.util.concurrent.TimeUnit;
69
70public class JingleRtpConnection extends AbstractJingleConnection
71 implements WebRTCWrapper.EventCallback, CallIntegration.Callback {
72
73 public static final List<State> STATES_SHOWING_ONGOING_CALL =
74 Arrays.asList(
75 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
76 private static final long BUSY_TIME_OUT = 30;
77
78 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
79 private final Queue<
80 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
81 pendingIceCandidates = new LinkedList<>();
82 private final OmemoVerification omemoVerification = new OmemoVerification();
83 private final CallIntegration callIntegration;
84 private final Message message;
85
86 private Set<Media> proposedMedia;
87 private RtpContentMap initiatorRtpContentMap;
88 private RtpContentMap responderRtpContentMap;
89 private RtpContentMap incomingContentAdd;
90 private RtpContentMap outgoingContentAdd;
91 private IceUdpTransportInfo.Setup peerDtlsSetup;
92 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
93 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
94 private ScheduledFuture<?> ringingTimeoutFuture;
95
96 JingleRtpConnection(
97 final JingleConnectionManager jingleConnectionManager,
98 final Id id,
99 final Jid initiator) {
100 this(
101 jingleConnectionManager,
102 id,
103 initiator,
104 new CallIntegration(
105 jingleConnectionManager
106 .getXmppConnectionService()
107 .getApplicationContext()));
108 this.callIntegration.setAddress(
109 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
110 this.callIntegration.setInitialized();
111 }
112
113 JingleRtpConnection(
114 final JingleConnectionManager jingleConnectionManager,
115 final Id id,
116 final Jid initiator,
117 final CallIntegration callIntegration) {
118 super(jingleConnectionManager, id, initiator);
119 final Conversation conversation =
120 jingleConnectionManager
121 .getXmppConnectionService()
122 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
123 this.message =
124 new Message(
125 conversation,
126 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
127 Message.TYPE_RTP_SESSION,
128 id.sessionId);
129 this.callIntegration = callIntegration;
130 this.callIntegration.setCallback(this);
131 }
132
133 @Override
134 synchronized void deliverPacket(final JinglePacket jinglePacket) {
135 switch (jinglePacket.getAction()) {
136 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
137 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
138 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
139 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
140 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
141 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
142 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
143 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
144 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
145 default -> {
146 respondOk(jinglePacket);
147 Log.d(
148 Config.LOGTAG,
149 String.format(
150 "%s: received unhandled jingle action %s",
151 id.account.getJid().asBareJid(), jinglePacket.getAction()));
152 }
153 }
154 }
155
156 @Override
157 synchronized void notifyRebound() {
158 if (isTerminated()) {
159 return;
160 }
161 webRTCWrapper.close();
162 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
163 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
164 }
165 if (isInState(
166 State.SESSION_INITIALIZED,
167 State.SESSION_INITIALIZED_PRE_APPROVED,
168 State.SESSION_ACCEPTED)) {
169 // we might have already changed resources (full jid) at this point; so this might not
170 // even reach the other party
171 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
172 } else {
173 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
174 finish();
175 }
176 }
177
178 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
179 respondOk(jinglePacket);
180 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
181 final State previous = this.state;
182 Log.d(
183 Config.LOGTAG,
184 id.account.getJid().asBareJid()
185 + ": received session terminate reason="
186 + wrapper.reason
187 + "("
188 + Strings.nullToEmpty(wrapper.text)
189 + ") while in state "
190 + previous);
191 if (TERMINATED.contains(previous)) {
192 Log.d(
193 Config.LOGTAG,
194 id.account.getJid().asBareJid()
195 + ": ignoring session terminate because already in "
196 + previous);
197 return;
198 }
199 webRTCWrapper.close();
200 final State target = reasonToState(wrapper.reason);
201 transitionOrThrow(target);
202 writeLogMessage(target);
203 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
204 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
205 }
206 finish();
207 }
208
209 private void receiveTransportInfo(final JinglePacket jinglePacket) {
210 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
211 // INITIALIZED only after transport-info has been received
212 if (isInState(
213 State.NULL,
214 State.PROCEED,
215 State.SESSION_INITIALIZED,
216 State.SESSION_INITIALIZED_PRE_APPROVED,
217 State.SESSION_ACCEPTED)) {
218 final RtpContentMap contentMap;
219 try {
220 contentMap = RtpContentMap.of(jinglePacket);
221 } catch (final IllegalArgumentException | NullPointerException e) {
222 Log.d(
223 Config.LOGTAG,
224 id.account.getJid().asBareJid()
225 + ": improperly formatted contents; ignoring",
226 e);
227 respondOk(jinglePacket);
228 return;
229 }
230 receiveTransportInfo(jinglePacket, contentMap);
231 } else {
232 if (isTerminated()) {
233 respondOk(jinglePacket);
234 Log.d(
235 Config.LOGTAG,
236 id.account.getJid().asBareJid()
237 + ": ignoring out-of-order transport info; we where already terminated");
238 } else {
239 Log.d(
240 Config.LOGTAG,
241 id.account.getJid().asBareJid()
242 + ": received transport info while in state="
243 + this.state);
244 terminateWithOutOfOrder(jinglePacket);
245 }
246 }
247 }
248
249 private void receiveTransportInfo(
250 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
251 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
252 candidates = contentMap.contents.entrySet();
253 final RtpContentMap remote = getRemoteContentMap();
254 final Set<String> remoteContentIds =
255 remote == null ? Collections.emptySet() : remote.contents.keySet();
256 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
257 Log.d(
258 Config.LOGTAG,
259 "received transport-info for unknown contents "
260 + contentMap.contents.keySet()
261 + " (known: "
262 + remoteContentIds
263 + ")");
264 respondOk(jinglePacket);
265 pendingIceCandidates.addAll(candidates);
266 return;
267 }
268 if (this.state != State.SESSION_ACCEPTED) {
269 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
270 respondOk(jinglePacket);
271 pendingIceCandidates.addAll(candidates);
272 return;
273 }
274 // zero candidates + modified credentials are an ICE restart offer
275 if (checkForIceRestart(jinglePacket, contentMap)) {
276 return;
277 }
278 respondOk(jinglePacket);
279 try {
280 processCandidates(candidates);
281 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
282 Log.w(
283 Config.LOGTAG,
284 id.account.getJid().asBareJid()
285 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
286 }
287 }
288
289 private void receiveContentAdd(final JinglePacket jinglePacket) {
290 final RtpContentMap modification;
291 try {
292 modification = RtpContentMap.of(jinglePacket);
293 modification.requireContentDescriptions();
294 } catch (final RuntimeException e) {
295 Log.d(
296 Config.LOGTAG,
297 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
298 Throwables.getRootCause(e));
299 respondOk(jinglePacket);
300 webRTCWrapper.close();
301 sendSessionTerminate(Reason.of(e), e.getMessage());
302 return;
303 }
304 if (isInState(State.SESSION_ACCEPTED)) {
305 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
306 final ListenableFuture<RtpContentMap> future =
307 receiveRtpContentMap(
308 modification,
309 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
310 Futures.addCallback(
311 future,
312 new FutureCallback<>() {
313 @Override
314 public void onSuccess(final RtpContentMap rtpContentMap) {
315 receiveContentAdd(jinglePacket, rtpContentMap);
316 }
317
318 @Override
319 public void onFailure(@NonNull Throwable throwable) {
320 respondOk(jinglePacket);
321 final Throwable rootCause = Throwables.getRootCause(throwable);
322 Log.d(
323 Config.LOGTAG,
324 id.account.getJid().asBareJid()
325 + ": improperly formatted contents in content-add",
326 throwable);
327 webRTCWrapper.close();
328 sendSessionTerminate(
329 Reason.ofThrowable(rootCause), rootCause.getMessage());
330 }
331 },
332 MoreExecutors.directExecutor());
333 } else {
334 terminateWithOutOfOrder(jinglePacket);
335 }
336 }
337
338 private void receiveContentAdd(
339 final JinglePacket jinglePacket, final RtpContentMap modification) {
340 final RtpContentMap remote = getRemoteContentMap();
341 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
342 respondOk(jinglePacket);
343 this.webRTCWrapper.close();
344 sendSessionTerminate(
345 Reason.FAILED_APPLICATION,
346 String.format(
347 "contents with names %s already exists",
348 Joiner.on(", ").join(modification.getNames())));
349 return;
350 }
351 final ContentAddition contentAddition =
352 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
353
354 final RtpContentMap outgoing = this.outgoingContentAdd;
355 final Set<ContentAddition.Summary> outgoingContentAddSummary =
356 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
357
358 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
359 if (isInitiator()) {
360 Log.d(
361 Config.LOGTAG,
362 id.getAccount().getJid().asBareJid()
363 + ": respond with tie break to matching content-add offer");
364 respondWithTieBreak(jinglePacket);
365 } else {
366 Log.d(
367 Config.LOGTAG,
368 id.getAccount().getJid().asBareJid()
369 + ": automatically accept matching content-add offer");
370 acceptContentAdd(contentAddition.summary, modification);
371 }
372 return;
373 }
374
375 // once we can display multiple video tracks we can be more loose with this condition
376 // theoretically it should also be fine to automatically accept audio only contents
377 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
381 this.incomingContentAdd = modification;
382 respondOk(jinglePacket);
383 updateEndUserState();
384 } else {
385 respondOk(jinglePacket);
386 // TODO do we want to add a reason?
387 rejectContentAdd(modification);
388 }
389 }
390
391 private void receiveContentAccept(final JinglePacket jinglePacket) {
392 final RtpContentMap receivedContentAccept;
393 try {
394 receivedContentAccept = RtpContentMap.of(jinglePacket);
395 receivedContentAccept.requireContentDescriptions();
396 } catch (final RuntimeException e) {
397 Log.d(
398 Config.LOGTAG,
399 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
400 Throwables.getRootCause(e));
401 respondOk(jinglePacket);
402 webRTCWrapper.close();
403 sendSessionTerminate(Reason.of(e), e.getMessage());
404 return;
405 }
406
407 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
408 if (outgoingContentAdd == null) {
409 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
410 terminateWithOutOfOrder(jinglePacket);
411 return;
412 }
413 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
414 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
415 this.outgoingContentAdd = null;
416 respondOk(jinglePacket);
417 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
418 final ListenableFuture<RtpContentMap> future =
419 receiveRtpContentMap(
420 receivedContentAccept,
421 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
422 Futures.addCallback(
423 future,
424 new FutureCallback<>() {
425 @Override
426 public void onSuccess(final RtpContentMap result) {
427 receiveContentAccept(result);
428 }
429
430 @Override
431 public void onFailure(@NonNull final Throwable throwable) {
432 webRTCWrapper.close();
433 sendSessionTerminate(
434 Reason.ofThrowable(throwable), throwable.getMessage());
435 }
436 },
437 MoreExecutors.directExecutor());
438 } else {
439 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
440 terminateWithOutOfOrder(jinglePacket);
441 }
442 }
443
444 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
445 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
446 final RtpContentMap modifiedContentMap =
447 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
448
449 setRemoteContentMap(modifiedContentMap);
450
451 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
452
453 final org.webrtc.SessionDescription sdp =
454 new org.webrtc.SessionDescription(
455 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
456
457 try {
458 this.webRTCWrapper.setRemoteDescription(sdp).get();
459 } catch (final Exception e) {
460 final Throwable cause = Throwables.getRootCause(e);
461 Log.d(
462 Config.LOGTAG,
463 id.getAccount().getJid().asBareJid()
464 + ": unable to set remote description after receiving content-accept",
465 cause);
466 webRTCWrapper.close();
467 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
468 return;
469 }
470 Log.d(
471 Config.LOGTAG,
472 id.getAccount().getJid().asBareJid()
473 + ": remote has accepted content-add "
474 + ContentAddition.summary(receivedContentAccept));
475 processCandidates(receivedContentAccept.contents.entrySet());
476 updateEndUserState();
477 }
478
479 private void receiveContentModify(final JinglePacket jinglePacket) {
480 if (this.state != State.SESSION_ACCEPTED) {
481 terminateWithOutOfOrder(jinglePacket);
482 return;
483 }
484 final Map<String, Content.Senders> modification =
485 Maps.transformEntries(
486 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
487 final boolean isInitiator = isInitiator();
488 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
489 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
490 final Set<String> currentOutgoingMediaIds =
491 currentOutgoing == null
492 ? Collections.emptySet()
493 : currentOutgoing.contents.keySet();
494 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
495 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
496 respondOk(jinglePacket);
497 final RtpContentMap modifiedContentMap;
498 try {
499 modifiedContentMap =
500 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
501 } catch (final IllegalArgumentException e) {
502 webRTCWrapper.close();
503 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
504 return;
505 }
506 this.outgoingContentAdd = modifiedContentMap;
507 Log.d(
508 Config.LOGTAG,
509 id.account.getJid().asBareJid()
510 + ": processed content-modification for pending content-add");
511 } else if (remoteContentMap != null
512 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
513 respondOk(jinglePacket);
514 final RtpContentMap modifiedRemoteContentMap;
515 try {
516 modifiedRemoteContentMap =
517 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
518 } catch (final IllegalArgumentException e) {
519 webRTCWrapper.close();
520 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
521 return;
522 }
523 final SessionDescription offer;
524 try {
525 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
526 } catch (final IllegalArgumentException | NullPointerException e) {
527 Log.d(
528 Config.LOGTAG,
529 id.getAccount().getJid().asBareJid()
530 + ": unable convert offer from content-modify to SDP",
531 e);
532 webRTCWrapper.close();
533 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
534 return;
535 }
536 Log.d(
537 Config.LOGTAG,
538 id.account.getJid().asBareJid() + ": auto accepting content-modification");
539 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
540 } else {
541 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
542 respondWithItemNotFound(jinglePacket);
543 }
544 }
545
546 private void autoAcceptContentModify(
547 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
548 this.setRemoteContentMap(modifiedRemoteContentMap);
549 final org.webrtc.SessionDescription sdp =
550 new org.webrtc.SessionDescription(
551 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
552 try {
553 this.webRTCWrapper.setRemoteDescription(sdp).get();
554 // auto accept is only done when we already have tracks
555 final SessionDescription answer = setLocalSessionDescription();
556 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
557 modifyLocalContentMap(rtpContentMap);
558 // we do not need to send an answer but do we have to resend the candidates currently in
559 // SDP?
560 // resendCandidatesFromSdp(answer);
561 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
562 } catch (final Exception e) {
563 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
564 webRTCWrapper.close();
565 sendSessionTerminate(Reason.FAILED_APPLICATION);
566 }
567 }
568
569 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
570 final SessionDescription answer) {
571 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
572 new ImmutableMultimap.Builder<>();
573 for (final SessionDescription.Media media : answer.media) {
574 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
575 if (Strings.isNullOrEmpty(mid)) {
576 continue;
577 }
578 for (final String sdpCandidate : media.attributes.get("candidate")) {
579 final IceUdpTransportInfo.Candidate candidate =
580 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
581 if (candidate != null) {
582 candidateBuilder.put(mid, candidate);
583 }
584 }
585 }
586 return candidateBuilder.build();
587 }
588
589 private void receiveContentReject(final JinglePacket jinglePacket) {
590 final RtpContentMap receivedContentReject;
591 try {
592 receivedContentReject = RtpContentMap.of(jinglePacket);
593 } catch (final RuntimeException e) {
594 Log.d(
595 Config.LOGTAG,
596 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
597 Throwables.getRootCause(e));
598 respondOk(jinglePacket);
599 this.webRTCWrapper.close();
600 sendSessionTerminate(Reason.of(e), e.getMessage());
601 return;
602 }
603
604 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
605 if (outgoingContentAdd == null) {
606 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
607 terminateWithOutOfOrder(jinglePacket);
608 return;
609 }
610 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
611 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
612 this.outgoingContentAdd = null;
613 respondOk(jinglePacket);
614 Log.d(Config.LOGTAG, jinglePacket.toString());
615 receiveContentReject(ourSummary);
616 } else {
617 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
618 terminateWithOutOfOrder(jinglePacket);
619 }
620 }
621
622 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
623 try {
624 this.webRTCWrapper.removeTrack(Media.VIDEO);
625 final RtpContentMap localContentMap = customRollback();
626 modifyLocalContentMap(localContentMap);
627 } catch (final Exception e) {
628 final Throwable cause = Throwables.getRootCause(e);
629 Log.d(
630 Config.LOGTAG,
631 id.getAccount().getJid().asBareJid()
632 + ": unable to rollback local description after receiving content-reject",
633 cause);
634 webRTCWrapper.close();
635 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
636 return;
637 }
638 Log.d(
639 Config.LOGTAG,
640 id.getAccount().getJid().asBareJid()
641 + ": remote has rejected our content-add "
642 + summary);
643 }
644
645 private void receiveContentRemove(final JinglePacket jinglePacket) {
646 final RtpContentMap receivedContentRemove;
647 try {
648 receivedContentRemove = RtpContentMap.of(jinglePacket);
649 receivedContentRemove.requireContentDescriptions();
650 } catch (final RuntimeException e) {
651 Log.d(
652 Config.LOGTAG,
653 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
654 Throwables.getRootCause(e));
655 respondOk(jinglePacket);
656 this.webRTCWrapper.close();
657 sendSessionTerminate(Reason.of(e), e.getMessage());
658 return;
659 }
660 respondOk(jinglePacket);
661 receiveContentRemove(receivedContentRemove);
662 }
663
664 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
665 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
666 final Set<ContentAddition.Summary> contentAddSummary =
667 incomingContentAdd == null
668 ? Collections.emptySet()
669 : ContentAddition.summary(incomingContentAdd);
670 final Set<ContentAddition.Summary> removeSummary =
671 ContentAddition.summary(receivedContentRemove);
672 if (contentAddSummary.equals(removeSummary)) {
673 this.incomingContentAdd = null;
674 updateEndUserState();
675 } else {
676 webRTCWrapper.close();
677 sendSessionTerminate(
678 Reason.FAILED_APPLICATION,
679 String.format(
680 "%s only supports %s as a means to retract a not yet accepted %s",
681 BuildConfig.APP_NAME,
682 JinglePacket.Action.CONTENT_REMOVE,
683 JinglePacket.Action.CONTENT_ADD));
684 }
685 }
686
687 public synchronized void retractContentAdd() {
688 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
689 if (outgoingContentAdd == null) {
690 throw new IllegalStateException("Not outgoing content add");
691 }
692 try {
693 webRTCWrapper.removeTrack(Media.VIDEO);
694 final RtpContentMap localContentMap = customRollback();
695 modifyLocalContentMap(localContentMap);
696 } catch (final Exception e) {
697 final Throwable cause = Throwables.getRootCause(e);
698 Log.d(
699 Config.LOGTAG,
700 id.getAccount().getJid().asBareJid()
701 + ": unable to rollback local description after trying to retract content-add",
702 cause);
703 webRTCWrapper.close();
704 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
705 return;
706 }
707 this.outgoingContentAdd = null;
708 final JinglePacket retract =
709 outgoingContentAdd
710 .toStub()
711 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
712 this.send(retract);
713 Log.d(
714 Config.LOGTAG,
715 id.getAccount().getJid()
716 + ": retract content-add "
717 + ContentAddition.summary(outgoingContentAdd));
718 }
719
720 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
721 final SessionDescription sdp = setLocalSessionDescription();
722 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
723 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
724 this.webRTCWrapper
725 .setRemoteDescription(
726 new org.webrtc.SessionDescription(
727 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
728 .get();
729 return localRtpContentMap;
730 }
731
732 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
733 final RtpContentMap currentRemote = getRemoteContentMap();
734 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
735 if (diff.isEmpty()) {
736 throw new IllegalStateException(
737 "Unexpected rollback condition. No difference between local and remote");
738 }
739 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
740 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
741 final RtpContentMap nextRemote =
742 currentRemote.addContent(
743 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
744 return SessionDescription.of(nextRemote, isResponder());
745 }
746 throw new IllegalStateException(
747 "Unexpected rollback condition. Senders were not uniformly none");
748 }
749
750 public synchronized void acceptContentAdd(
751 @NonNull final Set<ContentAddition.Summary> contentAddition) {
752 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
753 if (incomingContentAdd == null) {
754 throw new IllegalStateException("No incoming content add");
755 }
756
757 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
758 this.incomingContentAdd = null;
759 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
760 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
761 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
762 Log.d(
763 Config.LOGTAG,
764 "content addition is receive only. we want to upgrade to 'both'");
765 final RtpContentMap modifiedSenders =
766 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
767 final JinglePacket proposedContentModification =
768 modifiedSenders
769 .toStub()
770 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
771 proposedContentModification.setTo(id.with);
772 xmppConnectionService.sendIqPacket(
773 id.account,
774 proposedContentModification,
775 (account, response) -> {
776 if (response.getType() == IqPacket.TYPE.RESULT) {
777 Log.d(
778 Config.LOGTAG,
779 id.account.getJid().asBareJid()
780 + ": remote has accepted our upgrade to senders=both");
781 acceptContentAdd(
782 ContentAddition.summary(modifiedSenders), modifiedSenders);
783 } else {
784 Log.d(
785 Config.LOGTAG,
786 id.account.getJid().asBareJid()
787 + ": remote has rejected our upgrade to senders=both");
788 acceptContentAdd(contentAddition, incomingContentAdd);
789 }
790 });
791 } else {
792 acceptContentAdd(contentAddition, incomingContentAdd);
793 }
794 } else {
795 throw new IllegalStateException(
796 "Accepted content add does not match pending content-add");
797 }
798 }
799
800 private void acceptContentAdd(
801 @NonNull final Set<ContentAddition.Summary> contentAddition,
802 final RtpContentMap incomingContentAdd) {
803 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
804 final RtpContentMap modifiedContentMap =
805 getRemoteContentMap().addContent(incomingContentAdd, setup);
806 this.setRemoteContentMap(modifiedContentMap);
807
808 final SessionDescription offer;
809 try {
810 offer = SessionDescription.of(modifiedContentMap, isResponder());
811 } catch (final IllegalArgumentException | NullPointerException e) {
812 Log.d(
813 Config.LOGTAG,
814 id.getAccount().getJid().asBareJid()
815 + ": unable convert offer from content-add to SDP",
816 e);
817 webRTCWrapper.close();
818 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
819 return;
820 }
821 this.incomingContentAdd = null;
822 acceptContentAdd(contentAddition, offer);
823 }
824
825 private void acceptContentAdd(
826 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
827 final org.webrtc.SessionDescription sdp =
828 new org.webrtc.SessionDescription(
829 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
830 try {
831 this.webRTCWrapper.setRemoteDescription(sdp).get();
832
833 // TODO add tracks for 'media' where contentAddition.senders matches
834
835 // TODO if senders.sending(isInitiator())
836
837 this.webRTCWrapper.addTrack(Media.VIDEO);
838
839 // TODO add additional transceivers for recv only cases
840
841 final SessionDescription answer = setLocalSessionDescription();
842 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
843
844 final RtpContentMap contentAcceptMap =
845 rtpContentMap.toContentModification(
846 Collections2.transform(contentAddition, ca -> ca.name));
847
848 Log.d(
849 Config.LOGTAG,
850 id.getAccount().getJid().asBareJid()
851 + ": sending content-accept "
852 + ContentAddition.summary(contentAcceptMap));
853
854 addIceCandidatesFromBlackLog();
855
856 modifyLocalContentMap(rtpContentMap);
857 final ListenableFuture<RtpContentMap> future =
858 prepareOutgoingContentMap(contentAcceptMap);
859 Futures.addCallback(
860 future,
861 new FutureCallback<>() {
862 @Override
863 public void onSuccess(final RtpContentMap rtpContentMap) {
864 sendContentAccept(rtpContentMap);
865 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
866 }
867
868 @Override
869 public void onFailure(@NonNull final Throwable throwable) {
870 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
871 }
872 },
873 MoreExecutors.directExecutor());
874 } catch (final Exception e) {
875 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
876 webRTCWrapper.close();
877 sendSessionTerminate(Reason.FAILED_APPLICATION);
878 }
879 }
880
881 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
882 final JinglePacket jinglePacket =
883 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
884 send(jinglePacket);
885 }
886
887 public synchronized void rejectContentAdd() {
888 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
889 if (incomingContentAdd == null) {
890 throw new IllegalStateException("No incoming content add");
891 }
892 this.incomingContentAdd = null;
893 updateEndUserState();
894 rejectContentAdd(incomingContentAdd);
895 }
896
897 private void rejectContentAdd(final RtpContentMap contentMap) {
898 final JinglePacket jinglePacket =
899 contentMap
900 .toStub()
901 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
902 Log.d(
903 Config.LOGTAG,
904 id.getAccount().getJid().asBareJid()
905 + ": rejecting content "
906 + ContentAddition.summary(contentMap));
907 send(jinglePacket);
908 }
909
910 private boolean checkForIceRestart(
911 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
912 final RtpContentMap existing = getRemoteContentMap();
913 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
914 final IceUdpTransportInfo.Credentials newCredentials;
915 try {
916 existingCredentials = existing.getCredentials();
917 newCredentials = rtpContentMap.getDistinctCredentials();
918 } catch (final IllegalStateException e) {
919 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
920 return false;
921 }
922 if (existingCredentials.contains(newCredentials)) {
923 return false;
924 }
925 // TODO an alternative approach is to check if we already got an iq result to our
926 // ICE-restart
927 // and if that's the case we are seeing an answer.
928 // This might be more spec compliant but also more error prone potentially
929 final boolean isSignalStateStable =
930 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
931 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
932 final boolean isOffer = rtpContentMap.emptyCandidates();
933 final RtpContentMap restartContentMap;
934 try {
935 if (isOffer) {
936 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
937 restartContentMap =
938 existing.modifiedCredentials(
939 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
940 } else {
941 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
942 Log.d(
943 Config.LOGTAG,
944 "received confirmation of ICE restart"
945 + newCredentials
946 + " peer_setup="
947 + setup);
948 // DTLS setup attribute needs to be rewritten to reflect current peer state
949 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
950 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
951 }
952 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
953 return isOffer;
954 } else {
955 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
956 respondWithTieBreak(jinglePacket);
957 return true;
958 }
959 } catch (final Exception exception) {
960 respondOk(jinglePacket);
961 final Throwable rootCause = Throwables.getRootCause(exception);
962 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
963 // If this happens a termination is already in progress
964 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
965 return true;
966 }
967 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
968 webRTCWrapper.close();
969 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
970 return true;
971 }
972 }
973
974 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
975 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
976 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
977 throw new IllegalStateException("Invalid peer setup");
978 }
979 return peerSetup;
980 }
981
982 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
983 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
984 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
985 }
986 this.peerDtlsSetup = setup;
987 }
988
989 private boolean applyIceRestart(
990 final JinglePacket jinglePacket,
991 final RtpContentMap restartContentMap,
992 final boolean isOffer)
993 throws ExecutionException, InterruptedException {
994 final SessionDescription sessionDescription =
995 SessionDescription.of(restartContentMap, isResponder());
996 final org.webrtc.SessionDescription.Type type =
997 isOffer
998 ? org.webrtc.SessionDescription.Type.OFFER
999 : org.webrtc.SessionDescription.Type.ANSWER;
1000 org.webrtc.SessionDescription sdp =
1001 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1002 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1003 if (isInitiator()) {
1004 // We ignore the offer and respond with tie-break. This will clause the responder
1005 // not to apply the content map
1006 return false;
1007 }
1008 }
1009 webRTCWrapper.setRemoteDescription(sdp).get();
1010 setRemoteContentMap(restartContentMap);
1011 if (isOffer) {
1012 final SessionDescription localSessionDescription = setLocalSessionDescription();
1013 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1014 // We need to respond OK before sending any candidates
1015 respondOk(jinglePacket);
1016 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1017 } else {
1018 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1019 }
1020 return true;
1021 }
1022
1023 private void processCandidates(
1024 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1025 contents) {
1026 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1027 content : contents) {
1028 processCandidate(content);
1029 }
1030 }
1031
1032 private void processCandidate(
1033 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1034 content) {
1035 final RtpContentMap rtpContentMap = getRemoteContentMap();
1036 final List<String> indices = toIdentificationTags(rtpContentMap);
1037 final String sdpMid = content.getKey(); // aka content name
1038 final IceUdpTransportInfo transport = content.getValue().transport;
1039 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1040
1041 // TODO check that credentials remained the same
1042
1043 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1044 final String sdp;
1045 try {
1046 sdp = candidate.toSdpAttribute(credentials.ufrag);
1047 } catch (final IllegalArgumentException e) {
1048 Log.d(
1049 Config.LOGTAG,
1050 id.account.getJid().asBareJid()
1051 + ": ignoring invalid ICE candidate "
1052 + e.getMessage());
1053 continue;
1054 }
1055 final int mLineIndex = indices.indexOf(sdpMid);
1056 if (mLineIndex < 0) {
1057 Log.w(
1058 Config.LOGTAG,
1059 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1060 }
1061 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1062 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1063 this.webRTCWrapper.addIceCandidate(iceCandidate);
1064 }
1065 }
1066
1067 private RtpContentMap getRemoteContentMap() {
1068 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1069 }
1070
1071 private RtpContentMap getLocalContentMap() {
1072 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1073 }
1074
1075 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1076 final Group originalGroup = rtpContentMap.group;
1077 final List<String> identificationTags =
1078 originalGroup == null
1079 ? rtpContentMap.getNames()
1080 : originalGroup.getIdentificationTags();
1081 if (identificationTags.size() == 0) {
1082 Log.w(
1083 Config.LOGTAG,
1084 id.account.getJid().asBareJid()
1085 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1086 }
1087 return identificationTags;
1088 }
1089
1090 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1091 final JinglePacket jinglePacket, final boolean expectVerification) {
1092 try {
1093 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1094 } catch (final Exception e) {
1095 return Futures.immediateFailedFuture(e);
1096 }
1097 }
1098
1099 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1100 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1101 Log.d(
1102 Config.LOGTAG,
1103 "receiveRtpContentMap("
1104 + receivedContentMap.getClass().getSimpleName()
1105 + ",expectVerification="
1106 + expectVerification
1107 + ")");
1108 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1109 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1110 id.account
1111 .getAxolotlService()
1112 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1113 return Futures.transform(
1114 future,
1115 omemoVerifiedPayload -> {
1116 // TODO test if an exception here triggers a correct abort
1117 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1118 Log.d(
1119 Config.LOGTAG,
1120 id.account.getJid().asBareJid()
1121 + ": received verifiable DTLS fingerprint via "
1122 + omemoVerification);
1123 return omemoVerifiedPayload.getPayload();
1124 },
1125 MoreExecutors.directExecutor());
1126 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1127 return Futures.immediateFailedFuture(
1128 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1129 } else {
1130 return Futures.immediateFuture(receivedContentMap);
1131 }
1132 }
1133
1134 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1135 if (isInitiator()) {
1136 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1137 return;
1138 }
1139 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1140 Futures.addCallback(
1141 future,
1142 new FutureCallback<>() {
1143 @Override
1144 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1145 receiveSessionInitiate(jinglePacket, rtpContentMap);
1146 }
1147
1148 @Override
1149 public void onFailure(@NonNull final Throwable throwable) {
1150 respondOk(jinglePacket);
1151 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1152 }
1153 },
1154 MoreExecutors.directExecutor());
1155 }
1156
1157 private void receiveSessionInitiate(
1158 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1159 try {
1160 contentMap.requireContentDescriptions();
1161 contentMap.requireDTLSFingerprint(true);
1162 } catch (final RuntimeException e) {
1163 Log.d(
1164 Config.LOGTAG,
1165 id.account.getJid().asBareJid() + ": improperly formatted contents",
1166 Throwables.getRootCause(e));
1167 respondOk(jinglePacket);
1168 sendSessionTerminate(Reason.of(e), e.getMessage());
1169 return;
1170 }
1171 Log.d(
1172 Config.LOGTAG,
1173 "processing session-init with " + contentMap.contents.size() + " contents");
1174 final State target;
1175 if (this.state == State.PROCEED) {
1176 Preconditions.checkState(
1177 proposedMedia != null && proposedMedia.size() > 0,
1178 "proposed media must be set when processing pre-approved session-initiate");
1179 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1180 sendSessionTerminate(
1181 Reason.SECURITY_ERROR,
1182 String.format(
1183 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1184 this.proposedMedia, contentMap.getMedia()));
1185 return;
1186 }
1187 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1188 } else {
1189 target = State.SESSION_INITIALIZED;
1190 setProposedMedia(contentMap.getMedia());
1191 }
1192 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1193 respondOk(jinglePacket);
1194 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1195 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1196 Log.d(
1197 Config.LOGTAG,
1198 id.account.getJid().asBareJid()
1199 + ": automatically accepting session-initiate");
1200 sendSessionAccept();
1201 } else {
1202 Log.d(
1203 Config.LOGTAG,
1204 id.account.getJid().asBareJid()
1205 + ": received not pre-approved session-initiate. start ringing");
1206 startRinging();
1207 }
1208 } else {
1209 Log.d(
1210 Config.LOGTAG,
1211 String.format(
1212 "%s: received session-initiate while in state %s",
1213 id.account.getJid().asBareJid(), state));
1214 terminateWithOutOfOrder(jinglePacket);
1215 }
1216 }
1217
1218 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1219 if (isResponder()) {
1220 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1221 return;
1222 }
1223 final ListenableFuture<RtpContentMap> future =
1224 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1225 Futures.addCallback(
1226 future,
1227 new FutureCallback<>() {
1228 @Override
1229 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1230 receiveSessionAccept(jinglePacket, rtpContentMap);
1231 }
1232
1233 @Override
1234 public void onFailure(@NonNull final Throwable throwable) {
1235 respondOk(jinglePacket);
1236 Log.d(
1237 Config.LOGTAG,
1238 id.account.getJid().asBareJid()
1239 + ": improperly formatted contents in session-accept",
1240 throwable);
1241 webRTCWrapper.close();
1242 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1243 }
1244 },
1245 MoreExecutors.directExecutor());
1246 }
1247
1248 private void receiveSessionAccept(
1249 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1250 try {
1251 contentMap.requireContentDescriptions();
1252 contentMap.requireDTLSFingerprint();
1253 } catch (final RuntimeException e) {
1254 respondOk(jinglePacket);
1255 Log.d(
1256 Config.LOGTAG,
1257 id.account.getJid().asBareJid()
1258 + ": improperly formatted contents in session-accept",
1259 e);
1260 webRTCWrapper.close();
1261 sendSessionTerminate(Reason.of(e), e.getMessage());
1262 return;
1263 }
1264 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1265 if (!initiatorMedia.equals(contentMap.getMedia())) {
1266 sendSessionTerminate(
1267 Reason.SECURITY_ERROR,
1268 String.format(
1269 "Your session-included included media %s but our session-initiate was %s",
1270 this.proposedMedia, contentMap.getMedia()));
1271 return;
1272 }
1273 Log.d(
1274 Config.LOGTAG,
1275 "processing session-accept with " + contentMap.contents.size() + " contents");
1276 if (transition(State.SESSION_ACCEPTED)) {
1277 respondOk(jinglePacket);
1278 receiveSessionAccept(contentMap);
1279 } else {
1280 Log.d(
1281 Config.LOGTAG,
1282 String.format(
1283 "%s: received session-accept while in state %s",
1284 id.account.getJid().asBareJid(), state));
1285 respondOk(jinglePacket);
1286 }
1287 }
1288
1289 private void receiveSessionAccept(final RtpContentMap contentMap) {
1290 this.responderRtpContentMap = contentMap;
1291 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1292 final SessionDescription sessionDescription;
1293 try {
1294 sessionDescription = SessionDescription.of(contentMap, false);
1295 } catch (final IllegalArgumentException | NullPointerException e) {
1296 Log.d(
1297 Config.LOGTAG,
1298 id.account.getJid().asBareJid()
1299 + ": unable convert offer from session-accept to SDP",
1300 e);
1301 webRTCWrapper.close();
1302 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1303 return;
1304 }
1305 final org.webrtc.SessionDescription answer =
1306 new org.webrtc.SessionDescription(
1307 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1308 try {
1309 this.webRTCWrapper.setRemoteDescription(answer).get();
1310 } catch (final Exception e) {
1311 Log.d(
1312 Config.LOGTAG,
1313 id.account.getJid().asBareJid()
1314 + ": unable to set remote description after receiving session-accept",
1315 Throwables.getRootCause(e));
1316 webRTCWrapper.close();
1317 sendSessionTerminate(
1318 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1319 return;
1320 }
1321 processCandidates(contentMap.contents.entrySet());
1322 }
1323
1324 private void sendSessionAccept() {
1325 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1326 if (rtpContentMap == null) {
1327 throw new IllegalStateException("initiator RTP Content Map has not been set");
1328 }
1329 final SessionDescription offer;
1330 try {
1331 offer = SessionDescription.of(rtpContentMap, true);
1332 } catch (final IllegalArgumentException | NullPointerException e) {
1333 Log.d(
1334 Config.LOGTAG,
1335 id.account.getJid().asBareJid()
1336 + ": unable convert offer from session-initiate to SDP",
1337 e);
1338 webRTCWrapper.close();
1339 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1340 return;
1341 }
1342 sendSessionAccept(rtpContentMap.getMedia(), offer);
1343 }
1344
1345 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1346 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1347 }
1348
1349 private synchronized void sendSessionAccept(
1350 final Set<Media> media,
1351 final SessionDescription offer,
1352 final List<PeerConnection.IceServer> iceServers) {
1353 if (isTerminated()) {
1354 Log.w(
1355 Config.LOGTAG,
1356 id.account.getJid().asBareJid()
1357 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1358 return;
1359 }
1360 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1361 try {
1362 setupWebRTC(media, iceServers, !includeCandidates);
1363 } catch (final WebRTCWrapper.InitializationException e) {
1364 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1365 webRTCWrapper.close();
1366 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1367 return;
1368 }
1369 final org.webrtc.SessionDescription sdp =
1370 new org.webrtc.SessionDescription(
1371 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1372 try {
1373 this.webRTCWrapper.setRemoteDescription(sdp).get();
1374 addIceCandidatesFromBlackLog();
1375 org.webrtc.SessionDescription webRTCSessionDescription =
1376 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1377 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1378 } catch (final Exception e) {
1379 failureToAcceptSession(e);
1380 }
1381 }
1382
1383 private void failureToAcceptSession(final Throwable throwable) {
1384 if (isTerminated()) {
1385 return;
1386 }
1387 final Throwable rootCause = Throwables.getRootCause(throwable);
1388 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1389 webRTCWrapper.close();
1390 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1391 }
1392
1393 private void failureToPerformAction(
1394 final JinglePacket.Action action, final Throwable throwable) {
1395 if (isTerminated()) {
1396 return;
1397 }
1398 final Throwable rootCause = Throwables.getRootCause(throwable);
1399 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1400 webRTCWrapper.close();
1401 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1402 }
1403
1404 private void addIceCandidatesFromBlackLog() {
1405 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1406 while ((foo = this.pendingIceCandidates.poll()) != null) {
1407 processCandidate(foo);
1408 Log.d(
1409 Config.LOGTAG,
1410 id.account.getJid().asBareJid() + ": added candidate from back log");
1411 }
1412 }
1413
1414 private void prepareSessionAccept(
1415 final org.webrtc.SessionDescription webRTCSessionDescription,
1416 final boolean includeCandidates) {
1417 final SessionDescription sessionDescription =
1418 SessionDescription.parse(webRTCSessionDescription.description);
1419 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1420 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1421 if (includeCandidates) {
1422 candidates = parseCandidates(sessionDescription);
1423 } else {
1424 candidates = ImmutableMultimap.of();
1425 }
1426 this.responderRtpContentMap = respondingRtpContentMap;
1427 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1428 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1429 prepareOutgoingContentMap(respondingRtpContentMap);
1430 Futures.addCallback(
1431 outgoingContentMapFuture,
1432 new FutureCallback<>() {
1433 @Override
1434 public void onSuccess(final RtpContentMap outgoingContentMap) {
1435 if (includeCandidates) {
1436 Log.d(
1437 Config.LOGTAG,
1438 "including "
1439 + candidates.size()
1440 + " candidates in session accept");
1441 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1442 } else {
1443 sendSessionAccept(outgoingContentMap);
1444 }
1445 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1446 }
1447
1448 @Override
1449 public void onFailure(@NonNull Throwable throwable) {
1450 failureToAcceptSession(throwable);
1451 }
1452 },
1453 MoreExecutors.directExecutor());
1454 }
1455
1456 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1457 if (isTerminated()) {
1458 Log.w(
1459 Config.LOGTAG,
1460 id.account.getJid().asBareJid()
1461 + ": preparing session accept was too slow. already terminated. nothing to do.");
1462 return;
1463 }
1464 transitionOrThrow(State.SESSION_ACCEPTED);
1465 final JinglePacket sessionAccept =
1466 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1467 send(sessionAccept);
1468 }
1469
1470 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1471 final RtpContentMap rtpContentMap) {
1472 if (this.omemoVerification.hasDeviceId()) {
1473 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1474 verifiedPayloadFuture =
1475 id.account
1476 .getAxolotlService()
1477 .encrypt(
1478 rtpContentMap,
1479 id.with,
1480 omemoVerification.getDeviceId());
1481 return Futures.transform(
1482 verifiedPayloadFuture,
1483 verifiedPayload -> {
1484 omemoVerification.setOrEnsureEqual(verifiedPayload);
1485 return verifiedPayload.getPayload();
1486 },
1487 MoreExecutors.directExecutor());
1488 } else {
1489 return Futures.immediateFuture(rtpContentMap);
1490 }
1491 }
1492
1493 synchronized void deliveryMessage(
1494 final Jid from,
1495 final Element message,
1496 final String serverMessageId,
1497 final long timestamp) {
1498 Log.d(
1499 Config.LOGTAG,
1500 id.account.getJid().asBareJid()
1501 + ": delivered message to JingleRtpConnection "
1502 + message);
1503 switch (message.getName()) {
1504 case "propose" -> receivePropose(
1505 from, Propose.upgrade(message), serverMessageId, timestamp);
1506 case "proceed" -> receiveProceed(
1507 from, Proceed.upgrade(message), serverMessageId, timestamp);
1508 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1509 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1510 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1511 }
1512 }
1513
1514 void deliverFailedProceed(final String message) {
1515 Log.d(
1516 Config.LOGTAG,
1517 id.account.getJid().asBareJid()
1518 + ": receive message error for proceed message ("
1519 + Strings.nullToEmpty(message)
1520 + ")");
1521 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1522 webRTCWrapper.close();
1523 Log.d(
1524 Config.LOGTAG,
1525 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1526 this.finish();
1527 }
1528 }
1529
1530 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1531 final boolean originatedFromMyself =
1532 from.asBareJid().equals(id.account.getJid().asBareJid());
1533 if (originatedFromMyself) {
1534 if (transition(State.ACCEPTED)) {
1535 acceptedOnOtherDevice(serverMsgId, timestamp);
1536 } else {
1537 Log.d(
1538 Config.LOGTAG,
1539 id.account.getJid().asBareJid()
1540 + ": unable to transition to accept because already in state="
1541 + this.state);
1542 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1543 }
1544 } else {
1545 Log.d(
1546 Config.LOGTAG,
1547 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1548 }
1549 }
1550
1551 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1552 if (serverMsgId != null) {
1553 this.message.setServerMsgId(serverMsgId);
1554 }
1555 this.message.setTime(timestamp);
1556 this.message.setCarbon(true); // indicate that call was accepted on other device
1557 this.writeLogMessageSuccess(0);
1558 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1559 this.finish();
1560 }
1561
1562 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1563 final boolean originatedFromMyself =
1564 from.asBareJid().equals(id.account.getJid().asBareJid());
1565 // reject from another one of my clients
1566 if (originatedFromMyself) {
1567 receiveRejectFromMyself(serverMsgId, timestamp);
1568 } else if (isInitiator()) {
1569 if (from.equals(id.with)) {
1570 receiveRejectFromResponder();
1571 } else {
1572 Log.d(
1573 Config.LOGTAG,
1574 id.account.getJid()
1575 + ": ignoring reject from "
1576 + from
1577 + " for session with "
1578 + id.with);
1579 }
1580 } else {
1581 Log.d(
1582 Config.LOGTAG,
1583 id.account.getJid()
1584 + ": ignoring reject from "
1585 + from
1586 + " for session with "
1587 + id.with);
1588 }
1589 }
1590
1591 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1592 if (transition(State.REJECTED)) {
1593 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1594 this.finish();
1595 if (serverMsgId != null) {
1596 this.message.setServerMsgId(serverMsgId);
1597 }
1598 this.message.setTime(timestamp);
1599 this.message.setCarbon(true); // indicate that call was rejected on other device
1600 writeLogMessageMissed();
1601 } else {
1602 Log.d(
1603 Config.LOGTAG,
1604 "not able to transition into REJECTED because already in " + this.state);
1605 }
1606 }
1607
1608 private void receiveRejectFromResponder() {
1609 if (isInState(State.PROCEED)) {
1610 Log.d(
1611 Config.LOGTAG,
1612 id.account.getJid()
1613 + ": received reject while still in proceed. callee reconsidered");
1614 closeTransitionLogFinish(State.REJECTED_RACED);
1615 return;
1616 }
1617 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1622 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1623 return;
1624 }
1625 Log.d(
1626 Config.LOGTAG,
1627 id.account.getJid()
1628 + ": ignoring reject from responder because already in state "
1629 + this.state);
1630 }
1631
1632 private void receivePropose(
1633 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1634 final boolean originatedFromMyself =
1635 from.asBareJid().equals(id.account.getJid().asBareJid());
1636 if (originatedFromMyself) {
1637 Log.d(
1638 Config.LOGTAG,
1639 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1640 } else if (transition(
1641 State.PROPOSED,
1642 () -> {
1643 final Collection<RtpDescription> descriptions =
1644 Collections2.transform(
1645 Collections2.filter(
1646 propose.getDescriptions(),
1647 d -> d instanceof RtpDescription),
1648 input -> (RtpDescription) input);
1649 final Collection<Media> media =
1650 Collections2.transform(descriptions, RtpDescription::getMedia);
1651 Preconditions.checkState(
1652 !media.contains(Media.UNKNOWN),
1653 "RTP descriptions contain unknown media");
1654 Log.d(
1655 Config.LOGTAG,
1656 id.account.getJid().asBareJid()
1657 + ": received session proposal from "
1658 + from
1659 + " for "
1660 + media);
1661 this.setProposedMedia(Sets.newHashSet(media));
1662 })) {
1663 if (serverMsgId != null) {
1664 this.message.setServerMsgId(serverMsgId);
1665 }
1666 this.message.setTime(timestamp);
1667 startRinging();
1668 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1669 sendJingleMessage("ringing");
1670 }
1671 } else {
1672 Log.d(
1673 Config.LOGTAG,
1674 id.account.getJid()
1675 + ": ignoring session proposal because already in "
1676 + state);
1677 }
1678 }
1679
1680 private void startRinging() {
1681 this.callIntegration.setRinging();
1682 Log.d(
1683 Config.LOGTAG,
1684 id.account.getJid().asBareJid()
1685 + ": received call from "
1686 + id.with
1687 + ". start ringing");
1688 ringingTimeoutFuture =
1689 jingleConnectionManager.schedule(
1690 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1691 if (CallIntegration.selfManaged()) {
1692 return;
1693 }
1694 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1695 }
1696
1697 private synchronized void ringingTimeout() {
1698 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1699 switch (this.state) {
1700 case PROPOSED -> {
1701 message.markUnread();
1702 rejectCallFromProposed();
1703 }
1704 case SESSION_INITIALIZED -> {
1705 message.markUnread();
1706 rejectCallFromSessionInitiate();
1707 }
1708 }
1709 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1710 }
1711
1712 private void cancelRingingTimeout() {
1713 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1714 if (future != null && !future.isCancelled()) {
1715 future.cancel(false);
1716 }
1717 }
1718
1719 private void receiveProceed(
1720 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1721 final Set<Media> media =
1722 Preconditions.checkNotNull(
1723 this.proposedMedia, "Proposed media has to be set before handling proceed");
1724 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1725 if (from.equals(id.with)) {
1726 if (isInitiator()) {
1727 if (transition(State.PROCEED)) {
1728 if (serverMsgId != null) {
1729 this.message.setServerMsgId(serverMsgId);
1730 }
1731 this.message.setTime(timestamp);
1732 final Integer remoteDeviceId = proceed.getDeviceId();
1733 if (isOmemoEnabled()) {
1734 this.omemoVerification.setDeviceId(remoteDeviceId);
1735 } else {
1736 if (remoteDeviceId != null) {
1737 Log.d(
1738 Config.LOGTAG,
1739 id.account.getJid().asBareJid()
1740 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1741 }
1742 this.omemoVerification.setDeviceId(null);
1743 }
1744 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1745 } else {
1746 Log.d(
1747 Config.LOGTAG,
1748 String.format(
1749 "%s: ignoring proceed because already in %s",
1750 id.account.getJid().asBareJid(), this.state));
1751 }
1752 } else {
1753 Log.d(
1754 Config.LOGTAG,
1755 String.format(
1756 "%s: ignoring proceed because we were not initializing",
1757 id.account.getJid().asBareJid()));
1758 }
1759 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1760 if (transition(State.ACCEPTED)) {
1761 Log.d(
1762 Config.LOGTAG,
1763 id.account.getJid().asBareJid()
1764 + ": moved session with "
1765 + id.with
1766 + " into state accepted after received carbon copied proceed");
1767 acceptedOnOtherDevice(serverMsgId, timestamp);
1768 }
1769 } else {
1770 Log.d(
1771 Config.LOGTAG,
1772 String.format(
1773 "%s: ignoring proceed from %s. was expected from %s",
1774 id.account.getJid().asBareJid(), from, id.with));
1775 }
1776 }
1777
1778 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1779 if (from.equals(id.with)) {
1780 final State target =
1781 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1782 if (transition(target)) {
1783 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1784 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1785 Log.d(
1786 Config.LOGTAG,
1787 id.account.getJid().asBareJid()
1788 + ": session with "
1789 + id.with
1790 + " has been retracted (serverMsgId="
1791 + serverMsgId
1792 + ")");
1793 if (serverMsgId != null) {
1794 this.message.setServerMsgId(serverMsgId);
1795 }
1796 this.message.setTime(timestamp);
1797 if (target == State.RETRACTED) {
1798 this.message.markUnread();
1799 }
1800 writeLogMessageMissed();
1801 finish();
1802 } else {
1803 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1804 }
1805 } else {
1806 // TODO parse retract from self
1807 Log.d(
1808 Config.LOGTAG,
1809 id.account.getJid().asBareJid()
1810 + ": received retract from "
1811 + from
1812 + ". expected retract from"
1813 + id.with
1814 + ". ignoring");
1815 }
1816 }
1817
1818 public void sendSessionInitiate() {
1819 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1820 }
1821
1822 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1823 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1824 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1825 }
1826
1827 private synchronized void sendSessionInitiate(
1828 final Set<Media> media,
1829 final State targetState,
1830 final List<PeerConnection.IceServer> iceServers) {
1831 if (isTerminated()) {
1832 Log.w(
1833 Config.LOGTAG,
1834 id.account.getJid().asBareJid()
1835 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1836 return;
1837 }
1838 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1839 try {
1840 setupWebRTC(media, iceServers, !includeCandidates);
1841 } catch (final WebRTCWrapper.InitializationException e) {
1842 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1843 webRTCWrapper.close();
1844 sendRetract(Reason.ofThrowable(e));
1845 return;
1846 }
1847 try {
1848 org.webrtc.SessionDescription webRTCSessionDescription =
1849 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1850 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1851 } catch (final Exception e) {
1852 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1853 // exceptions
1854 failureToInitiateSession(e, targetState);
1855 }
1856 }
1857
1858 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1859 if (isTerminated()) {
1860 return;
1861 }
1862 Log.d(
1863 Config.LOGTAG,
1864 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1865 Throwables.getRootCause(throwable));
1866 webRTCWrapper.close();
1867 final Reason reason = Reason.ofThrowable(throwable);
1868 if (isInState(targetState)) {
1869 sendSessionTerminate(reason, throwable.getMessage());
1870 } else {
1871 sendRetract(reason);
1872 }
1873 }
1874
1875 private void sendRetract(final Reason reason) {
1876 // TODO embed reason into retract
1877 sendJingleMessage("retract", id.with.asBareJid());
1878 transitionOrThrow(reasonToState(reason));
1879 this.finish();
1880 }
1881
1882 private void prepareSessionInitiate(
1883 final org.webrtc.SessionDescription webRTCSessionDescription,
1884 final boolean includeCandidates,
1885 final State targetState) {
1886 final SessionDescription sessionDescription =
1887 SessionDescription.parse(webRTCSessionDescription.description);
1888 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1889 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1890 if (includeCandidates) {
1891 candidates = parseCandidates(sessionDescription);
1892 } else {
1893 candidates = ImmutableMultimap.of();
1894 }
1895 this.initiatorRtpContentMap = rtpContentMap;
1896 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1897 encryptSessionInitiate(rtpContentMap);
1898 Futures.addCallback(
1899 outgoingContentMapFuture,
1900 new FutureCallback<>() {
1901 @Override
1902 public void onSuccess(final RtpContentMap outgoingContentMap) {
1903 if (includeCandidates) {
1904 Log.d(
1905 Config.LOGTAG,
1906 "including "
1907 + candidates.size()
1908 + " candidates in session initiate");
1909 sendSessionInitiate(
1910 outgoingContentMap.withCandidates(candidates), targetState);
1911 } else {
1912 sendSessionInitiate(outgoingContentMap, targetState);
1913 }
1914 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1915 }
1916
1917 @Override
1918 public void onFailure(@NonNull final Throwable throwable) {
1919 failureToInitiateSession(throwable, targetState);
1920 }
1921 },
1922 MoreExecutors.directExecutor());
1923 }
1924
1925 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1926 if (isTerminated()) {
1927 Log.w(
1928 Config.LOGTAG,
1929 id.account.getJid().asBareJid()
1930 + ": preparing session was too slow. already terminated. nothing to do.");
1931 return;
1932 }
1933 this.transitionOrThrow(targetState);
1934 final JinglePacket sessionInitiate =
1935 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1936 send(sessionInitiate);
1937 }
1938
1939 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1940 final RtpContentMap rtpContentMap) {
1941 if (this.omemoVerification.hasDeviceId()) {
1942 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1943 verifiedPayloadFuture =
1944 id.account
1945 .getAxolotlService()
1946 .encrypt(
1947 rtpContentMap,
1948 id.with,
1949 omemoVerification.getDeviceId());
1950 final ListenableFuture<RtpContentMap> future =
1951 Futures.transform(
1952 verifiedPayloadFuture,
1953 verifiedPayload -> {
1954 omemoVerification.setSessionFingerprint(
1955 verifiedPayload.getFingerprint());
1956 return verifiedPayload.getPayload();
1957 },
1958 MoreExecutors.directExecutor());
1959 if (Config.REQUIRE_RTP_VERIFICATION) {
1960 return future;
1961 }
1962 return Futures.catching(
1963 future,
1964 CryptoFailedException.class,
1965 e -> {
1966 Log.w(
1967 Config.LOGTAG,
1968 id.account.getJid().asBareJid()
1969 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1970 e);
1971 return rtpContentMap;
1972 },
1973 MoreExecutors.directExecutor());
1974 } else {
1975 return Futures.immediateFuture(rtpContentMap);
1976 }
1977 }
1978
1979 protected void sendSessionTerminate(final Reason reason) {
1980 sendSessionTerminate(reason, null);
1981 }
1982
1983 protected void sendSessionTerminate(final Reason reason, final String text) {
1984 sendSessionTerminate(reason, text, this::writeLogMessage);
1985 }
1986
1987 private void sendTransportInfo(
1988 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1989 final RtpContentMap transportInfo;
1990 try {
1991 final RtpContentMap rtpContentMap =
1992 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1993 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1994 } catch (final Exception e) {
1995 Log.d(
1996 Config.LOGTAG,
1997 id.account.getJid().asBareJid()
1998 + ": unable to prepare transport-info from candidate for content="
1999 + contentName);
2000 return;
2001 }
2002 final JinglePacket jinglePacket =
2003 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2004 send(jinglePacket);
2005 }
2006
2007 public RtpEndUserState getEndUserState() {
2008 switch (this.state) {
2009 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2010 if (isInitiator()) {
2011 return RtpEndUserState.RINGING;
2012 } else {
2013 return RtpEndUserState.INCOMING_CALL;
2014 }
2015 }
2016 case PROCEED -> {
2017 if (isInitiator()) {
2018 return RtpEndUserState.RINGING;
2019 } else {
2020 return RtpEndUserState.ACCEPTING_CALL;
2021 }
2022 }
2023 case SESSION_INITIALIZED_PRE_APPROVED -> {
2024 if (isInitiator()) {
2025 return RtpEndUserState.RINGING;
2026 } else {
2027 return RtpEndUserState.CONNECTING;
2028 }
2029 }
2030 case SESSION_ACCEPTED -> {
2031 final ContentAddition ca = getPendingContentAddition();
2032 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2033 return RtpEndUserState.INCOMING_CONTENT_ADD;
2034 }
2035 return getPeerConnectionStateAsEndUserState();
2036 }
2037 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.DECLINED_OR_BUSY;
2040 } else {
2041 return RtpEndUserState.ENDED;
2042 }
2043 }
2044 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2045 return RtpEndUserState.ENDED;
2046 }
2047 case RETRACTED_RACED -> {
2048 if (isInitiator()) {
2049 return RtpEndUserState.ENDED;
2050 } else {
2051 return RtpEndUserState.RETRACTED;
2052 }
2053 }
2054 case TERMINATED_CONNECTIVITY_ERROR -> {
2055 return zeroDuration()
2056 ? RtpEndUserState.CONNECTIVITY_ERROR
2057 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2058 }
2059 case TERMINATED_APPLICATION_FAILURE -> {
2060 return RtpEndUserState.APPLICATION_ERROR;
2061 }
2062 case TERMINATED_SECURITY_ERROR -> {
2063 return RtpEndUserState.SECURITY_ERROR;
2064 }
2065 }
2066 throw new IllegalStateException(
2067 String.format("%s has no equivalent EndUserState", this.state));
2068 }
2069
2070 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2071 final PeerConnection.PeerConnectionState state;
2072 try {
2073 state = webRTCWrapper.getState();
2074 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2075 // We usually close the WebRTCWrapper *before* transitioning so we might still
2076 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2077 return RtpEndUserState.ENDING_CALL;
2078 }
2079 return switch (state) {
2080 case CONNECTED -> RtpEndUserState.CONNECTED;
2081 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2082 case CLOSED -> RtpEndUserState.ENDING_CALL;
2083 default -> zeroDuration()
2084 ? RtpEndUserState.CONNECTIVITY_ERROR
2085 : RtpEndUserState.RECONNECTING;
2086 };
2087 }
2088
2089 private boolean isPeerConnectionConnected() {
2090 try {
2091 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2092 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2093 return false;
2094 }
2095 }
2096
2097 private void updateCallIntegrationState() {
2098 switch (this.state) {
2099 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2100 if (isInitiator()) {
2101 this.callIntegration.setDialing();
2102 } else {
2103 this.callIntegration.setRinging();
2104 }
2105 }
2106 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2107 if (isInitiator()) {
2108 this.callIntegration.setDialing();
2109 } else {
2110 this.callIntegration.setInitialized();
2111 }
2112 }
2113 case SESSION_ACCEPTED -> {
2114 if (isPeerConnectionConnected()) {
2115 this.callIntegration.setActive();
2116 } else {
2117 this.callIntegration.setInitialized();
2118 }
2119 }
2120 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2121 if (isInitiator()) {
2122 this.callIntegration.busy();
2123 } else {
2124 this.callIntegration.rejected();
2125 }
2126 }
2127 case TERMINATED_SUCCESS -> this.callIntegration.success();
2128 case ACCEPTED -> this.callIntegration.accepted();
2129 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2130 .retracted();
2131 case TERMINATED_CONNECTIVITY_ERROR,
2132 TERMINATED_APPLICATION_FAILURE,
2133 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2134 default -> throw new IllegalStateException(
2135 String.format("%s is not handled", this.state));
2136 }
2137 }
2138
2139 public ContentAddition getPendingContentAddition() {
2140 final RtpContentMap in = this.incomingContentAdd;
2141 final RtpContentMap out = this.outgoingContentAdd;
2142 if (out != null) {
2143 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2144 } else if (in != null) {
2145 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2146 } else {
2147 return null;
2148 }
2149 }
2150
2151 public Set<Media> getMedia() {
2152 final State current = getState();
2153 if (current == State.NULL) {
2154 if (isInitiator()) {
2155 return Preconditions.checkNotNull(
2156 this.proposedMedia, "RTP connection has not been initialized properly");
2157 }
2158 throw new IllegalStateException("RTP connection has not been initialized yet");
2159 }
2160 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2161 return Preconditions.checkNotNull(
2162 this.proposedMedia, "RTP connection has not been initialized properly");
2163 }
2164 final RtpContentMap localContentMap = getLocalContentMap();
2165 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2166 if (localContentMap != null) {
2167 return localContentMap.getMedia();
2168 } else if (initiatorContentMap != null) {
2169 return initiatorContentMap.getMedia();
2170 } else if (isTerminated()) {
2171 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2172 } else {
2173 return Preconditions.checkNotNull(
2174 this.proposedMedia, "RTP connection has not been initialized properly");
2175 }
2176 }
2177
2178 public boolean isVerified() {
2179 final String fingerprint = this.omemoVerification.getFingerprint();
2180 if (fingerprint == null) {
2181 return false;
2182 }
2183 final FingerprintStatus status =
2184 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2185 return status != null && status.isVerified();
2186 }
2187
2188 public boolean addMedia(final Media media) {
2189 final Set<Media> currentMedia = getMedia();
2190 if (currentMedia.contains(media)) {
2191 throw new IllegalStateException(String.format("%s has already been proposed", media));
2192 }
2193 // TODO add state protection - can only add while ACCEPTED or so
2194 Log.d(Config.LOGTAG, "adding media: " + media);
2195 return webRTCWrapper.addTrack(media);
2196 }
2197
2198 public synchronized void acceptCall() {
2199 switch (this.state) {
2200 case PROPOSED -> {
2201 cancelRingingTimeout();
2202 acceptCallFromProposed();
2203 }
2204 case SESSION_INITIALIZED -> {
2205 cancelRingingTimeout();
2206 acceptCallFromSessionInitialized();
2207 }
2208 case ACCEPTED -> Log.w(
2209 Config.LOGTAG,
2210 id.account.getJid().asBareJid()
2211 + ": the call has already been accepted with another client. UI was just lagging behind");
2212 case PROCEED, SESSION_ACCEPTED -> Log.w(
2213 Config.LOGTAG,
2214 id.account.getJid().asBareJid()
2215 + ": the call has already been accepted. user probably double tapped the UI");
2216 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2217 }
2218 }
2219
2220 public synchronized void rejectCall() {
2221 if (isTerminated()) {
2222 Log.w(
2223 Config.LOGTAG,
2224 id.account.getJid().asBareJid()
2225 + ": received rejectCall() when session has already been terminated. nothing to do");
2226 return;
2227 }
2228 switch (this.state) {
2229 case PROPOSED -> rejectCallFromProposed();
2230 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2231 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2232 }
2233 }
2234
2235 public synchronized void endCall() {
2236 if (isTerminated()) {
2237 Log.w(
2238 Config.LOGTAG,
2239 id.account.getJid().asBareJid()
2240 + ": received endCall() when session has already been terminated. nothing to do");
2241 return;
2242 }
2243 if (isInState(State.PROPOSED) && isResponder()) {
2244 rejectCallFromProposed();
2245 return;
2246 }
2247 if (isInState(State.PROCEED)) {
2248 if (isInitiator()) {
2249 retractFromProceed();
2250 } else {
2251 rejectCallFromProceed();
2252 }
2253 return;
2254 }
2255 if (isInitiator()
2256 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2257 this.webRTCWrapper.close();
2258 sendSessionTerminate(Reason.CANCEL);
2259 return;
2260 }
2261 if (isInState(State.SESSION_INITIALIZED)) {
2262 rejectCallFromSessionInitiate();
2263 return;
2264 }
2265 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2266 this.webRTCWrapper.close();
2267 sendSessionTerminate(Reason.SUCCESS);
2268 return;
2269 }
2270 if (isInState(
2271 State.TERMINATED_APPLICATION_FAILURE,
2272 State.TERMINATED_CONNECTIVITY_ERROR,
2273 State.TERMINATED_DECLINED_OR_BUSY)) {
2274 Log.d(
2275 Config.LOGTAG,
2276 "ignoring request to end call because already in state " + this.state);
2277 return;
2278 }
2279 throw new IllegalStateException(
2280 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2281 }
2282
2283 private void retractFromProceed() {
2284 Log.d(Config.LOGTAG, "retract from proceed");
2285 this.sendJingleMessage("retract");
2286 closeTransitionLogFinish(State.RETRACTED_RACED);
2287 }
2288
2289 private void closeTransitionLogFinish(final State state) {
2290 this.webRTCWrapper.close();
2291 transitionOrThrow(state);
2292 writeLogMessage(state);
2293 finish();
2294 }
2295
2296 private void setupWebRTC(
2297 final Set<Media> media,
2298 final List<PeerConnection.IceServer> iceServers,
2299 final boolean trickle)
2300 throws WebRTCWrapper.InitializationException {
2301 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2302 this.webRTCWrapper.setup(
2303 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2304 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2305 }
2306
2307 private void acceptCallFromProposed() {
2308 transitionOrThrow(State.PROCEED);
2309 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2310 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2311 this.sendJingleMessage("proceed");
2312 }
2313
2314 private void rejectCallFromProposed() {
2315 transitionOrThrow(State.REJECTED);
2316 writeLogMessageMissed();
2317 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2318 this.sendJingleMessage("reject");
2319 finish();
2320 }
2321
2322 private void rejectCallFromProceed() {
2323 this.sendJingleMessage("reject");
2324 closeTransitionLogFinish(State.REJECTED_RACED);
2325 }
2326
2327 private void rejectCallFromSessionInitiate() {
2328 webRTCWrapper.close();
2329 sendSessionTerminate(Reason.DECLINE);
2330 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2331 }
2332
2333 private void sendJingleMessage(final String action) {
2334 sendJingleMessage(action, id.with);
2335 }
2336
2337 private void sendJingleMessage(final String action, final Jid to) {
2338 final MessagePacket messagePacket = new MessagePacket();
2339 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2340 messagePacket.setTo(to);
2341 final Element intent =
2342 messagePacket
2343 .addChild(action, Namespace.JINGLE_MESSAGE)
2344 .setAttribute("id", id.sessionId);
2345 if ("proceed".equals(action)) {
2346 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2347 if (isOmemoEnabled()) {
2348 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2349 final Element device =
2350 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2351 device.setAttribute("id", deviceId);
2352 }
2353 }
2354 messagePacket.addChild("store", "urn:xmpp:hints");
2355 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2356 }
2357
2358 private boolean isOmemoEnabled() {
2359 final Conversational conversational = message.getConversation();
2360 if (conversational instanceof Conversation) {
2361 return ((Conversation) conversational).getNextEncryption()
2362 == Message.ENCRYPTION_AXOLOTL;
2363 }
2364 return false;
2365 }
2366
2367 private void acceptCallFromSessionInitialized() {
2368 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2369 sendSessionAccept();
2370 }
2371
2372 @Override
2373 protected synchronized boolean transition(final State target, final Runnable runnable) {
2374 if (super.transition(target, runnable)) {
2375 updateEndUserState();
2376 updateOngoingCallNotification();
2377 return true;
2378 } else {
2379 return false;
2380 }
2381 }
2382
2383 @Override
2384 public void onIceCandidate(final IceCandidate iceCandidate) {
2385 final RtpContentMap rtpContentMap =
2386 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2387 final IceUdpTransportInfo.Credentials credentials;
2388 try {
2389 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2390 } catch (final IllegalArgumentException e) {
2391 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2392 return;
2393 }
2394 final String uFrag = credentials.ufrag;
2395 final IceUdpTransportInfo.Candidate candidate =
2396 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2397 if (candidate == null) {
2398 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2399 return;
2400 }
2401 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2402 sendTransportInfo(iceCandidate.sdpMid, candidate);
2403 }
2404
2405 @Override
2406 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2407 Log.d(
2408 Config.LOGTAG,
2409 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2410 this.stateHistory.add(newState);
2411 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2412 this.sessionDuration.start();
2413 updateOngoingCallNotification();
2414 } else if (this.sessionDuration.isRunning()) {
2415 this.sessionDuration.stop();
2416 updateOngoingCallNotification();
2417 }
2418
2419 final boolean neverConnected =
2420 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2421
2422 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2423 if (neverConnected) {
2424 if (isTerminated()) {
2425 Log.d(
2426 Config.LOGTAG,
2427 id.account.getJid().asBareJid()
2428 + ": not sending session-terminate after connectivity error because session is already in state "
2429 + this.state);
2430 return;
2431 }
2432 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2433 return;
2434 } else {
2435 this.restartIce();
2436 }
2437 }
2438 updateEndUserState();
2439 }
2440
2441 private void restartIce() {
2442 this.stateHistory.clear();
2443 this.webRTCWrapper.restartIceAsync();
2444 }
2445
2446 @Override
2447 public void onRenegotiationNeeded() {
2448 this.webRTCWrapper.execute(this::renegotiate);
2449 }
2450
2451 private void renegotiate() {
2452 final SessionDescription sessionDescription;
2453 try {
2454 sessionDescription = setLocalSessionDescription();
2455 } catch (final Exception e) {
2456 final Throwable cause = Throwables.getRootCause(e);
2457 webRTCWrapper.close();
2458 if (isTerminated()) {
2459 Log.d(
2460 Config.LOGTAG,
2461 "failed to renegotiate. session was already terminated",
2462 cause);
2463 return;
2464 }
2465 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2466 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2467 return;
2468 }
2469 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2470 final RtpContentMap currentContentMap = getLocalContentMap();
2471 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2472 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2473
2474 Log.d(
2475 Config.LOGTAG,
2476 id.getAccount().getJid().asBareJid()
2477 + ": renegotiate. iceRestart="
2478 + iceRestart
2479 + " content id diff="
2480 + diff);
2481
2482 if (diff.hasModifications() && iceRestart) {
2483 webRTCWrapper.close();
2484 sendSessionTerminate(
2485 Reason.FAILED_APPLICATION,
2486 "WebRTC unexpectedly tried to modify content and transport at once");
2487 return;
2488 }
2489
2490 if (iceRestart) {
2491 initiateIceRestart(rtpContentMap);
2492 return;
2493 } else if (diff.isEmpty()) {
2494 Log.d(
2495 Config.LOGTAG,
2496 "renegotiation. nothing to do. SignalingState="
2497 + this.webRTCWrapper.getSignalingState());
2498 }
2499
2500 if (diff.added.size() > 0) {
2501 modifyLocalContentMap(rtpContentMap);
2502 sendContentAdd(rtpContentMap, diff.added);
2503 }
2504 }
2505
2506 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2507 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2508 final JinglePacket jinglePacket =
2509 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2510 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2511 jinglePacket.setTo(id.with);
2512 xmppConnectionService.sendIqPacket(
2513 id.account,
2514 jinglePacket,
2515 (account, response) -> {
2516 if (response.getType() == IqPacket.TYPE.RESULT) {
2517 Log.d(Config.LOGTAG, "received success to our ice restart");
2518 setLocalContentMap(rtpContentMap);
2519 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2520 return;
2521 }
2522 if (response.getType() == IqPacket.TYPE.ERROR) {
2523 if (isTieBreak(response)) {
2524 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2525 return;
2526 }
2527 handleIqErrorResponse(response);
2528 }
2529 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2530 handleIqTimeoutResponse(response);
2531 }
2532 });
2533 }
2534
2535 private boolean isTieBreak(final IqPacket response) {
2536 final Element error = response.findChild("error");
2537 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2538 }
2539
2540 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2541 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2542 this.outgoingContentAdd = contentAdd;
2543 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2544 prepareOutgoingContentMap(contentAdd);
2545 Futures.addCallback(
2546 outgoingContentMapFuture,
2547 new FutureCallback<>() {
2548 @Override
2549 public void onSuccess(final RtpContentMap outgoingContentMap) {
2550 sendContentAdd(outgoingContentMap);
2551 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2552 }
2553
2554 @Override
2555 public void onFailure(@NonNull Throwable throwable) {
2556 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2557 }
2558 },
2559 MoreExecutors.directExecutor());
2560 }
2561
2562 private void sendContentAdd(final RtpContentMap contentAdd) {
2563
2564 final JinglePacket jinglePacket =
2565 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2566 jinglePacket.setTo(id.with);
2567 xmppConnectionService.sendIqPacket(
2568 id.account,
2569 jinglePacket,
2570 (connection, response) -> {
2571 if (response.getType() == IqPacket.TYPE.RESULT) {
2572 Log.d(
2573 Config.LOGTAG,
2574 id.getAccount().getJid().asBareJid()
2575 + ": received ACK to our content-add");
2576 return;
2577 }
2578 if (response.getType() == IqPacket.TYPE.ERROR) {
2579 if (isTieBreak(response)) {
2580 this.outgoingContentAdd = null;
2581 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2582 return;
2583 }
2584 handleIqErrorResponse(response);
2585 }
2586 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2587 handleIqTimeoutResponse(response);
2588 }
2589 });
2590 }
2591
2592 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2593 if (isInitiator()) {
2594 this.initiatorRtpContentMap = rtpContentMap;
2595 } else {
2596 this.responderRtpContentMap = rtpContentMap;
2597 }
2598 }
2599
2600 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2601 if (isInitiator()) {
2602 this.responderRtpContentMap = rtpContentMap;
2603 } else {
2604 this.initiatorRtpContentMap = rtpContentMap;
2605 }
2606 }
2607
2608 // this method is to be used for content map modifications that modify media
2609 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2610 final RtpContentMap activeContents = rtpContentMap.activeContents();
2611 setLocalContentMap(activeContents);
2612 updateEndUserState();
2613 }
2614
2615 private SessionDescription setLocalSessionDescription()
2616 throws ExecutionException, InterruptedException {
2617 final org.webrtc.SessionDescription sessionDescription =
2618 this.webRTCWrapper.setLocalDescription(false).get();
2619 return SessionDescription.parse(sessionDescription.description);
2620 }
2621
2622 private void closeWebRTCSessionAfterFailedConnection() {
2623 this.webRTCWrapper.close();
2624 synchronized (this) {
2625 if (isTerminated()) {
2626 Log.d(
2627 Config.LOGTAG,
2628 id.account.getJid().asBareJid()
2629 + ": no need to send session-terminate after failed connection. Other party already did");
2630 return;
2631 }
2632 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2633 }
2634 }
2635
2636 public boolean zeroDuration() {
2637 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2638 }
2639
2640 public long getCallDuration() {
2641 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2642 }
2643
2644 public CallIntegration getCallIntegration() {
2645 return this.callIntegration;
2646 }
2647
2648 public boolean isMicrophoneEnabled() {
2649 return webRTCWrapper.isMicrophoneEnabled();
2650 }
2651
2652 public boolean setMicrophoneEnabled(final boolean enabled) {
2653 return webRTCWrapper.setMicrophoneEnabled(enabled);
2654 }
2655
2656 public boolean isVideoEnabled() {
2657 return webRTCWrapper.isVideoEnabled();
2658 }
2659
2660 public void setVideoEnabled(final boolean enabled) {
2661 webRTCWrapper.setVideoEnabled(enabled);
2662 }
2663
2664 public boolean isCameraSwitchable() {
2665 return webRTCWrapper.isCameraSwitchable();
2666 }
2667
2668 public boolean isFrontCamera() {
2669 return webRTCWrapper.isFrontCamera();
2670 }
2671
2672 public ListenableFuture<Boolean> switchCamera() {
2673 return webRTCWrapper.switchCamera();
2674 }
2675
2676 @Override
2677 public void onCallIntegrationShowIncomingCallUi() {
2678 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2679 }
2680
2681 @Override
2682 public void onCallIntegrationDisconnect() {
2683 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2684 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2685 rejectCall();
2686 } else {
2687 endCall();
2688 }
2689 }
2690
2691 @Override
2692 public void onCallIntegrationReject() {
2693 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2694 try {
2695 rejectCall();
2696 } catch (final IllegalStateException e) {
2697 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2698 }
2699 }
2700
2701 @Override
2702 public void onCallIntegrationAnswer() {
2703 // we need to start the UI to a) show it and b) be able to ask for permissions
2704 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2705 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2706 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2707 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2708 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2709 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2710 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2711 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2712 xmppConnectionService.startActivity(intent);
2713 }
2714
2715 @Override
2716 public void onAudioDeviceChanged(
2717 final CallIntegration.AudioDevice selectedAudioDevice,
2718 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2719 Log.d(
2720 Config.LOGTAG,
2721 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2722 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2723 selectedAudioDevice, availableAudioDevices);
2724 }
2725
2726 private void updateEndUserState() {
2727 final RtpEndUserState endUserState = getEndUserState();
2728 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2729 this.updateCallIntegrationState();
2730 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2731 id.account, id.with, id.sessionId, endUserState);
2732 }
2733
2734 private void updateOngoingCallNotification() {
2735 final State state = this.state;
2736 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2737 final boolean reconnecting;
2738 if (state == State.SESSION_ACCEPTED) {
2739 reconnecting =
2740 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2741 } else {
2742 reconnecting = false;
2743 }
2744 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2745 } else {
2746 xmppConnectionService.removeOngoingCall();
2747 }
2748 }
2749
2750 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2751 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2752 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2753 request.setTo(id.account.getDomain());
2754 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2755 xmppConnectionService.sendIqPacket(
2756 id.account,
2757 request,
2758 (account, response) -> {
2759 final var iceServers = IceServers.parse(response);
2760 if (iceServers.size() == 0) {
2761 Log.w(
2762 Config.LOGTAG,
2763 id.account.getJid().asBareJid()
2764 + ": no ICE server found "
2765 + response);
2766 }
2767 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2768 });
2769 } else {
2770 Log.w(
2771 Config.LOGTAG,
2772 id.account.getJid().asBareJid() + ": has no external service discovery");
2773 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2774 }
2775 }
2776
2777 @Override
2778 protected void terminateTransport() {
2779 this.webRTCWrapper.close();
2780 }
2781
2782 @Override
2783 protected void finish() {
2784 if (isTerminated()) {
2785 this.cancelRingingTimeout();
2786 this.callIntegration.verifyDisconnected();
2787 this.webRTCWrapper.verifyClosed();
2788 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2789 super.finish();
2790 } else {
2791 throw new IllegalStateException(
2792 String.format("Unable to call finish from %s", this.state));
2793 }
2794 }
2795
2796 private void writeLogMessage(final State state) {
2797 final long duration = getCallDuration();
2798 if (state == State.TERMINATED_SUCCESS
2799 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2800 writeLogMessageSuccess(duration);
2801 } else {
2802 writeLogMessageMissed();
2803 }
2804 }
2805
2806 private void writeLogMessageSuccess(final long duration) {
2807 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2808 this.writeMessage();
2809 }
2810
2811 private void writeLogMessageMissed() {
2812 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2813 this.writeMessage();
2814 }
2815
2816 private void writeMessage() {
2817 final Conversational conversational = message.getConversation();
2818 if (conversational instanceof Conversation) {
2819 ((Conversation) conversational).add(this.message);
2820 xmppConnectionService.createMessageAsync(message);
2821 xmppConnectionService.updateConversationUi();
2822 } else {
2823 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2824 }
2825 }
2826
2827 public Optional<VideoTrack> getLocalVideoTrack() {
2828 return webRTCWrapper.getLocalVideoTrack();
2829 }
2830
2831 public Optional<VideoTrack> getRemoteVideoTrack() {
2832 return webRTCWrapper.getRemoteVideoTrack();
2833 }
2834
2835 public EglBase.Context getEglBaseContext() {
2836 return webRTCWrapper.getEglBaseContext();
2837 }
2838
2839 void setProposedMedia(final Set<Media> media) {
2840 this.proposedMedia = media;
2841 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2842 }
2843
2844 public void fireStateUpdate() {
2845 final RtpEndUserState endUserState = getEndUserState();
2846 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2847 id.account, id.with, id.sessionId, endUserState);
2848 }
2849
2850 public boolean isSwitchToVideoAvailable() {
2851 final boolean prerequisite =
2852 Media.audioOnly(getMedia())
2853 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2854 .contains(getEndUserState());
2855 return prerequisite && remoteHasVideoFeature();
2856 }
2857
2858 private boolean remoteHasVideoFeature() {
2859 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2860 }
2861
2862 private boolean remoteHasSdpOfferAnswer() {
2863 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2864 }
2865
2866 private interface OnIceServersDiscovered {
2867 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2868 }
2869}