1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
51import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
52
53import im.conversations.android.xmpp.model.jingle.Jingle;
54import im.conversations.android.xmpp.model.stanza.Iq;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 final var contact = id.getContact();
122 this.callIntegration.setCallerDisplayName(
123 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
124 this.callIntegration.setInitialized();
125 }
126
127 JingleRtpConnection(
128 final JingleConnectionManager jingleConnectionManager,
129 final Id id,
130 final Jid initiator,
131 final CallIntegration callIntegration) {
132 super(jingleConnectionManager, id, initiator);
133 final Conversation conversation =
134 jingleConnectionManager
135 .getXmppConnectionService()
136 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
137 this.message =
138 new Message(
139 conversation,
140 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
141 Message.TYPE_RTP_SESSION,
142 id.sessionId);
143 this.callIntegration = callIntegration;
144 this.callIntegration.setCallback(this);
145 }
146
147 @Override
148 synchronized void deliverPacket(final Iq iq) {
149 final var jingle = iq.getExtension(Jingle.class);
150 switch (jingle.getAction()) {
151 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
152 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
153 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
154 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
155 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
156 case CONTENT_ACCEPT -> receiveContentAccept(iq);
157 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
158 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
159 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
160 default -> {
161 respondOk(iq);
162 Log.d(
163 Config.LOGTAG,
164 String.format(
165 "%s: received unhandled jingle action %s",
166 id.account.getJid().asBareJid(), jingle.getAction()));
167 }
168 }
169 }
170
171 @Override
172 synchronized void notifyRebound() {
173 if (isTerminated()) {
174 return;
175 }
176 webRTCWrapper.close();
177 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
178 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
179 }
180 if (isInState(
181 State.SESSION_INITIALIZED,
182 State.SESSION_INITIALIZED_PRE_APPROVED,
183 State.SESSION_ACCEPTED)) {
184 // we might have already changed resources (full jid) at this point; so this might not
185 // even reach the other party
186 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
187 } else {
188 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
189 finish();
190 }
191 }
192
193 public boolean applyDtmfTone(final String tone) {
194 return webRTCWrapper.applyDtmfTone(tone);
195 }
196
197 private void receiveSessionTerminate(final Iq jinglePacket) {
198 respondOk(jinglePacket);
199 final var jingle = jinglePacket.getExtension(Jingle.class);
200 final Jingle.ReasonWrapper wrapper = jingle.getReason();
201 final State previous = this.state;
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": received session terminate reason="
206 + wrapper.reason
207 + "("
208 + Strings.nullToEmpty(wrapper.text)
209 + ") while in state "
210 + previous);
211 if (TERMINATED.contains(previous)) {
212 Log.d(
213 Config.LOGTAG,
214 id.account.getJid().asBareJid()
215 + ": ignoring session terminate because already in "
216 + previous);
217 return;
218 }
219 webRTCWrapper.close();
220 final State target = reasonToState(wrapper.reason);
221 transitionOrThrow(target);
222 writeLogMessage(target);
223 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 finish();
227 }
228
229 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
230 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
231 // INITIALIZED only after transport-info has been received
232 if (isInState(
233 State.NULL,
234 State.PROCEED,
235 State.SESSION_INITIALIZED,
236 State.SESSION_INITIALIZED_PRE_APPROVED,
237 State.SESSION_ACCEPTED)) {
238 final RtpContentMap contentMap;
239 try {
240 contentMap = RtpContentMap.of(jingle);
241 } catch (final IllegalArgumentException | NullPointerException e) {
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": improperly formatted contents; ignoring",
246 e);
247 respondOk(jinglePacket);
248 return;
249 }
250 receiveTransportInfo(jinglePacket, contentMap);
251 } else {
252 if (isTerminated()) {
253 respondOk(jinglePacket);
254 Log.d(
255 Config.LOGTAG,
256 id.account.getJid().asBareJid()
257 + ": ignoring out-of-order transport info; we where already terminated");
258 } else {
259 Log.d(
260 Config.LOGTAG,
261 id.account.getJid().asBareJid()
262 + ": received transport info while in state="
263 + this.state);
264 terminateWithOutOfOrder(jinglePacket);
265 }
266 }
267 }
268
269 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
270 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
271 candidates = contentMap.contents.entrySet();
272 final RtpContentMap remote = getRemoteContentMap();
273 final Set<String> remoteContentIds =
274 remote == null ? Collections.emptySet() : remote.contents.keySet();
275 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
276 Log.d(
277 Config.LOGTAG,
278 "received transport-info for unknown contents "
279 + contentMap.contents.keySet()
280 + " (known: "
281 + remoteContentIds
282 + ")");
283 respondOk(jinglePacket);
284 pendingIceCandidates.addAll(candidates);
285 return;
286 }
287 if (this.state != State.SESSION_ACCEPTED) {
288 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
289 respondOk(jinglePacket);
290 pendingIceCandidates.addAll(candidates);
291 return;
292 }
293 // zero candidates + modified credentials are an ICE restart offer
294 if (checkForIceRestart(jinglePacket, contentMap)) {
295 return;
296 }
297 respondOk(jinglePacket);
298 try {
299 processCandidates(candidates);
300 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
301 Log.w(
302 Config.LOGTAG,
303 id.account.getJid().asBareJid()
304 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
305 }
306 }
307
308 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
309 final RtpContentMap modification;
310 try {
311 modification = RtpContentMap.of(jingle);
312 modification.requireContentDescriptions();
313 } catch (final RuntimeException e) {
314 Log.d(
315 Config.LOGTAG,
316 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
317 Throwables.getRootCause(e));
318 respondOk(iq);
319 webRTCWrapper.close();
320 sendSessionTerminate(Reason.of(e), e.getMessage());
321 return;
322 }
323 if (isInState(State.SESSION_ACCEPTED)) {
324 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
325 final ListenableFuture<RtpContentMap> future =
326 receiveRtpContentMap(
327 modification,
328 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
329 Futures.addCallback(
330 future,
331 new FutureCallback<>() {
332 @Override
333 public void onSuccess(final RtpContentMap rtpContentMap) {
334 receiveContentAdd(iq, rtpContentMap);
335 }
336
337 @Override
338 public void onFailure(@NonNull Throwable throwable) {
339 respondOk(iq);
340 final Throwable rootCause = Throwables.getRootCause(throwable);
341 Log.d(
342 Config.LOGTAG,
343 id.account.getJid().asBareJid()
344 + ": improperly formatted contents in content-add",
345 throwable);
346 webRTCWrapper.close();
347 sendSessionTerminate(
348 Reason.ofThrowable(rootCause), rootCause.getMessage());
349 }
350 },
351 MoreExecutors.directExecutor());
352 } else {
353 terminateWithOutOfOrder(iq);
354 }
355 }
356
357 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
358 final RtpContentMap remote = getRemoteContentMap();
359 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
360 respondOk(jinglePacket);
361 this.webRTCWrapper.close();
362 sendSessionTerminate(
363 Reason.FAILED_APPLICATION,
364 String.format(
365 "contents with names %s already exists",
366 Joiner.on(", ").join(modification.getNames())));
367 return;
368 }
369 final ContentAddition contentAddition =
370 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
371
372 final RtpContentMap outgoing = this.outgoingContentAdd;
373 final Set<ContentAddition.Summary> outgoingContentAddSummary =
374 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
375
376 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
377 if (isInitiator()) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid()
381 + ": respond with tie break to matching content-add offer");
382 respondWithTieBreak(jinglePacket);
383 } else {
384 Log.d(
385 Config.LOGTAG,
386 id.getAccount().getJid().asBareJid()
387 + ": automatically accept matching content-add offer");
388 acceptContentAdd(contentAddition.summary, modification);
389 }
390 return;
391 }
392
393 // once we can display multiple video tracks we can be more loose with this condition
394 // theoretically it should also be fine to automatically accept audio only contents
395 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
396 Log.d(
397 Config.LOGTAG,
398 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
399 this.incomingContentAdd = modification;
400 respondOk(jinglePacket);
401 updateEndUserState();
402 } else {
403 respondOk(jinglePacket);
404 // TODO do we want to add a reason?
405 rejectContentAdd(modification);
406 }
407 }
408
409 private void receiveContentAccept(final Iq jinglePacket) {
410 final var jingle = jinglePacket.getExtension(Jingle.class);
411 final RtpContentMap receivedContentAccept;
412 try {
413 receivedContentAccept = RtpContentMap.of(jingle);
414 receivedContentAccept.requireContentDescriptions();
415 } catch (final RuntimeException e) {
416 Log.d(
417 Config.LOGTAG,
418 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
419 Throwables.getRootCause(e));
420 respondOk(jinglePacket);
421 webRTCWrapper.close();
422 sendSessionTerminate(Reason.of(e), e.getMessage());
423 return;
424 }
425
426 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
427 if (outgoingContentAdd == null) {
428 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
429 terminateWithOutOfOrder(jinglePacket);
430 return;
431 }
432 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
433 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
434 this.outgoingContentAdd = null;
435 respondOk(jinglePacket);
436 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
437 final ListenableFuture<RtpContentMap> future =
438 receiveRtpContentMap(
439 receivedContentAccept,
440 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
441 Futures.addCallback(
442 future,
443 new FutureCallback<>() {
444 @Override
445 public void onSuccess(final RtpContentMap result) {
446 receiveContentAccept(result);
447 }
448
449 @Override
450 public void onFailure(@NonNull final Throwable throwable) {
451 webRTCWrapper.close();
452 sendSessionTerminate(
453 Reason.ofThrowable(throwable), throwable.getMessage());
454 }
455 },
456 MoreExecutors.directExecutor());
457 } else {
458 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
459 terminateWithOutOfOrder(jinglePacket);
460 }
461 }
462
463 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
464 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
465 final RtpContentMap modifiedContentMap =
466 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
467
468 setRemoteContentMap(modifiedContentMap);
469
470 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
471
472 final org.webrtc.SessionDescription sdp =
473 new org.webrtc.SessionDescription(
474 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
475
476 try {
477 this.webRTCWrapper.setRemoteDescription(sdp).get();
478 } catch (final Exception e) {
479 final Throwable cause = Throwables.getRootCause(e);
480 Log.d(
481 Config.LOGTAG,
482 id.getAccount().getJid().asBareJid()
483 + ": unable to set remote description after receiving content-accept",
484 cause);
485 webRTCWrapper.close();
486 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
487 return;
488 }
489 Log.d(
490 Config.LOGTAG,
491 id.getAccount().getJid().asBareJid()
492 + ": remote has accepted content-add "
493 + ContentAddition.summary(receivedContentAccept));
494 processCandidates(receivedContentAccept.contents.entrySet());
495 updateEndUserState();
496 }
497
498 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
499 if (this.state != State.SESSION_ACCEPTED) {
500 terminateWithOutOfOrder(jinglePacket);
501 return;
502 }
503 final Map<String, Content.Senders> modification =
504 Maps.transformEntries(
505 jingle.getJingleContents(), (key, value) -> value.getSenders());
506 final boolean isInitiator = isInitiator();
507 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
508 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
509 final Set<String> currentOutgoingMediaIds =
510 currentOutgoing == null
511 ? Collections.emptySet()
512 : currentOutgoing.contents.keySet();
513 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
514 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
515 respondOk(jinglePacket);
516 final RtpContentMap modifiedContentMap;
517 try {
518 modifiedContentMap =
519 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
520 } catch (final IllegalArgumentException e) {
521 webRTCWrapper.close();
522 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
523 return;
524 }
525 this.outgoingContentAdd = modifiedContentMap;
526 Log.d(
527 Config.LOGTAG,
528 id.account.getJid().asBareJid()
529 + ": processed content-modification for pending content-add");
530 } else if (remoteContentMap != null
531 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
532 respondOk(jinglePacket);
533 final RtpContentMap modifiedRemoteContentMap;
534 try {
535 modifiedRemoteContentMap =
536 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
537 } catch (final IllegalArgumentException e) {
538 webRTCWrapper.close();
539 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
540 return;
541 }
542 final SessionDescription offer;
543 try {
544 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
545 } catch (final IllegalArgumentException | NullPointerException e) {
546 Log.d(
547 Config.LOGTAG,
548 id.getAccount().getJid().asBareJid()
549 + ": unable convert offer from content-modify to SDP",
550 e);
551 webRTCWrapper.close();
552 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
553 return;
554 }
555 Log.d(
556 Config.LOGTAG,
557 id.account.getJid().asBareJid() + ": auto accepting content-modification");
558 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
559 } else {
560 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
561 respondWithItemNotFound(jinglePacket);
562 }
563 }
564
565 private void autoAcceptContentModify(
566 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
567 this.setRemoteContentMap(modifiedRemoteContentMap);
568 final org.webrtc.SessionDescription sdp =
569 new org.webrtc.SessionDescription(
570 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
571 try {
572 this.webRTCWrapper.setRemoteDescription(sdp).get();
573 // auto accept is only done when we already have tracks
574 final SessionDescription answer = setLocalSessionDescription();
575 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
576 modifyLocalContentMap(rtpContentMap);
577 // we do not need to send an answer but do we have to resend the candidates currently in
578 // SDP?
579 // resendCandidatesFromSdp(answer);
580 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
581 } catch (final Exception e) {
582 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
583 webRTCWrapper.close();
584 sendSessionTerminate(Reason.FAILED_APPLICATION);
585 }
586 }
587
588 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
589 final SessionDescription answer) {
590 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
591 new ImmutableMultimap.Builder<>();
592 for (final SessionDescription.Media media : answer.media) {
593 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
594 if (Strings.isNullOrEmpty(mid)) {
595 continue;
596 }
597 for (final String sdpCandidate : media.attributes.get("candidate")) {
598 final IceUdpTransportInfo.Candidate candidate =
599 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
600 if (candidate != null) {
601 candidateBuilder.put(mid, candidate);
602 }
603 }
604 }
605 return candidateBuilder.build();
606 }
607
608 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
609 final RtpContentMap receivedContentReject;
610 try {
611 receivedContentReject = RtpContentMap.of(jingle);
612 } catch (final RuntimeException e) {
613 Log.d(
614 Config.LOGTAG,
615 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
616 Throwables.getRootCause(e));
617 respondOk(jinglePacket);
618 this.webRTCWrapper.close();
619 sendSessionTerminate(Reason.of(e), e.getMessage());
620 return;
621 }
622
623 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
624 if (outgoingContentAdd == null) {
625 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
626 terminateWithOutOfOrder(jinglePacket);
627 return;
628 }
629 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
630 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
631 this.outgoingContentAdd = null;
632 respondOk(jinglePacket);
633 Log.d(Config.LOGTAG, jinglePacket.toString());
634 receiveContentReject(ourSummary);
635 } else {
636 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
637 terminateWithOutOfOrder(jinglePacket);
638 }
639 }
640
641 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
642 try {
643 this.webRTCWrapper.removeTrack(Media.VIDEO);
644 final RtpContentMap localContentMap = customRollback();
645 modifyLocalContentMap(localContentMap);
646 } catch (final Exception e) {
647 final Throwable cause = Throwables.getRootCause(e);
648 Log.d(
649 Config.LOGTAG,
650 id.getAccount().getJid().asBareJid()
651 + ": unable to rollback local description after receiving content-reject",
652 cause);
653 webRTCWrapper.close();
654 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
655 return;
656 }
657 Log.d(
658 Config.LOGTAG,
659 id.getAccount().getJid().asBareJid()
660 + ": remote has rejected our content-add "
661 + summary);
662 }
663
664 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
665 final RtpContentMap receivedContentRemove;
666 try {
667 receivedContentRemove = RtpContentMap.of(jingle);
668 receivedContentRemove.requireContentDescriptions();
669 } catch (final RuntimeException e) {
670 Log.d(
671 Config.LOGTAG,
672 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
673 Throwables.getRootCause(e));
674 respondOk(jinglePacket);
675 this.webRTCWrapper.close();
676 sendSessionTerminate(Reason.of(e), e.getMessage());
677 return;
678 }
679 respondOk(jinglePacket);
680 receiveContentRemove(receivedContentRemove);
681 }
682
683 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
684 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
685 final Set<ContentAddition.Summary> contentAddSummary =
686 incomingContentAdd == null
687 ? Collections.emptySet()
688 : ContentAddition.summary(incomingContentAdd);
689 final Set<ContentAddition.Summary> removeSummary =
690 ContentAddition.summary(receivedContentRemove);
691 if (contentAddSummary.equals(removeSummary)) {
692 this.incomingContentAdd = null;
693 updateEndUserState();
694 } else {
695 webRTCWrapper.close();
696 sendSessionTerminate(
697 Reason.FAILED_APPLICATION,
698 String.format(
699 "%s only supports %s as a means to retract a not yet accepted %s",
700 BuildConfig.APP_NAME,
701 Jingle.Action.CONTENT_REMOVE,
702 Jingle.Action.CONTENT_ADD));
703 }
704 }
705
706 public synchronized void retractContentAdd() {
707 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
708 if (outgoingContentAdd == null) {
709 throw new IllegalStateException("Not outgoing content add");
710 }
711 try {
712 webRTCWrapper.removeTrack(Media.VIDEO);
713 final RtpContentMap localContentMap = customRollback();
714 modifyLocalContentMap(localContentMap);
715 } catch (final Exception e) {
716 final Throwable cause = Throwables.getRootCause(e);
717 Log.d(
718 Config.LOGTAG,
719 id.getAccount().getJid().asBareJid()
720 + ": unable to rollback local description after trying to retract content-add",
721 cause);
722 webRTCWrapper.close();
723 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
724 return;
725 }
726 this.outgoingContentAdd = null;
727 final Iq retract =
728 outgoingContentAdd
729 .toStub()
730 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
731 this.send(retract);
732 Log.d(
733 Config.LOGTAG,
734 id.getAccount().getJid()
735 + ": retract content-add "
736 + ContentAddition.summary(outgoingContentAdd));
737 }
738
739 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
740 final SessionDescription sdp = setLocalSessionDescription();
741 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
742 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
743 this.webRTCWrapper
744 .setRemoteDescription(
745 new org.webrtc.SessionDescription(
746 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
747 .get();
748 return localRtpContentMap;
749 }
750
751 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
752 final RtpContentMap currentRemote = getRemoteContentMap();
753 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
754 if (diff.isEmpty()) {
755 throw new IllegalStateException(
756 "Unexpected rollback condition. No difference between local and remote");
757 }
758 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
759 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
760 final RtpContentMap nextRemote =
761 currentRemote.addContent(
762 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
763 return SessionDescription.of(nextRemote, isResponder());
764 }
765 throw new IllegalStateException(
766 "Unexpected rollback condition. Senders were not uniformly none");
767 }
768
769 public synchronized void acceptContentAdd(
770 @NonNull final Set<ContentAddition.Summary> contentAddition) {
771 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
772 if (incomingContentAdd == null) {
773 throw new IllegalStateException("No incoming content add");
774 }
775
776 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
777 this.incomingContentAdd = null;
778 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
779 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
780 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
781 Log.d(
782 Config.LOGTAG,
783 "content addition is receive only. we want to upgrade to 'both'");
784 final RtpContentMap modifiedSenders =
785 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
786 final Iq proposedContentModification =
787 modifiedSenders
788 .toStub()
789 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
790 proposedContentModification.setTo(id.with);
791 xmppConnectionService.sendIqPacket(
792 id.account,
793 proposedContentModification,
794 (response) -> {
795 if (response.getType() == Iq.Type.RESULT) {
796 Log.d(
797 Config.LOGTAG,
798 id.account.getJid().asBareJid()
799 + ": remote has accepted our upgrade to senders=both");
800 acceptContentAdd(
801 ContentAddition.summary(modifiedSenders), modifiedSenders);
802 } else {
803 Log.d(
804 Config.LOGTAG,
805 id.account.getJid().asBareJid()
806 + ": remote has rejected our upgrade to senders=both");
807 acceptContentAdd(contentAddition, incomingContentAdd);
808 }
809 });
810 } else {
811 acceptContentAdd(contentAddition, incomingContentAdd);
812 }
813 } else {
814 throw new IllegalStateException(
815 "Accepted content add does not match pending content-add");
816 }
817 }
818
819 private void acceptContentAdd(
820 @NonNull final Set<ContentAddition.Summary> contentAddition,
821 final RtpContentMap incomingContentAdd) {
822 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
823 final RtpContentMap modifiedContentMap =
824 getRemoteContentMap().addContent(incomingContentAdd, setup);
825 this.setRemoteContentMap(modifiedContentMap);
826
827 final SessionDescription offer;
828 try {
829 offer = SessionDescription.of(modifiedContentMap, isResponder());
830 } catch (final IllegalArgumentException | NullPointerException e) {
831 Log.d(
832 Config.LOGTAG,
833 id.getAccount().getJid().asBareJid()
834 + ": unable convert offer from content-add to SDP",
835 e);
836 webRTCWrapper.close();
837 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
838 return;
839 }
840 this.incomingContentAdd = null;
841 acceptContentAdd(contentAddition, offer);
842 }
843
844 private void acceptContentAdd(
845 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
846 final org.webrtc.SessionDescription sdp =
847 new org.webrtc.SessionDescription(
848 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
849 try {
850 this.webRTCWrapper.setRemoteDescription(sdp).get();
851
852 // TODO add tracks for 'media' where contentAddition.senders matches
853
854 // TODO if senders.sending(isInitiator())
855
856 this.webRTCWrapper.addTrack(Media.VIDEO);
857
858 // TODO add additional transceivers for recv only cases
859
860 final SessionDescription answer = setLocalSessionDescription();
861 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
862
863 final RtpContentMap contentAcceptMap =
864 rtpContentMap.toContentModification(
865 Collections2.transform(contentAddition, ca -> ca.name));
866
867 Log.d(
868 Config.LOGTAG,
869 id.getAccount().getJid().asBareJid()
870 + ": sending content-accept "
871 + ContentAddition.summary(contentAcceptMap));
872
873 addIceCandidatesFromBlackLog();
874
875 modifyLocalContentMap(rtpContentMap);
876 final ListenableFuture<RtpContentMap> future =
877 prepareOutgoingContentMap(contentAcceptMap);
878 Futures.addCallback(
879 future,
880 new FutureCallback<>() {
881 @Override
882 public void onSuccess(final RtpContentMap rtpContentMap) {
883 sendContentAccept(rtpContentMap);
884 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
885 }
886
887 @Override
888 public void onFailure(@NonNull final Throwable throwable) {
889 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
890 }
891 },
892 MoreExecutors.directExecutor());
893 } catch (final Exception e) {
894 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
895 webRTCWrapper.close();
896 sendSessionTerminate(Reason.FAILED_APPLICATION);
897 }
898 }
899
900 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
901 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
902 send(iq);
903 }
904
905 public synchronized void rejectContentAdd() {
906 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
907 if (incomingContentAdd == null) {
908 throw new IllegalStateException("No incoming content add");
909 }
910 this.incomingContentAdd = null;
911 updateEndUserState();
912 rejectContentAdd(incomingContentAdd);
913 }
914
915 private void rejectContentAdd(final RtpContentMap contentMap) {
916 final Iq iq =
917 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
918 Log.d(
919 Config.LOGTAG,
920 id.getAccount().getJid().asBareJid()
921 + ": rejecting content "
922 + ContentAddition.summary(contentMap));
923 send(iq);
924 }
925
926 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
927 final RtpContentMap existing = getRemoteContentMap();
928 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
929 final IceUdpTransportInfo.Credentials newCredentials;
930 try {
931 existingCredentials = existing.getCredentials();
932 newCredentials = rtpContentMap.getDistinctCredentials();
933 } catch (final IllegalStateException e) {
934 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
935 return false;
936 }
937 if (existingCredentials.contains(newCredentials)) {
938 return false;
939 }
940 // TODO an alternative approach is to check if we already got an iq result to our
941 // ICE-restart
942 // and if that's the case we are seeing an answer.
943 // This might be more spec compliant but also more error prone potentially
944 final boolean isSignalStateStable =
945 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
946 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
947 final boolean isOffer = rtpContentMap.emptyCandidates();
948 final RtpContentMap restartContentMap;
949 try {
950 if (isOffer) {
951 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
952 restartContentMap =
953 existing.modifiedCredentials(
954 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
955 } else {
956 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
957 Log.d(
958 Config.LOGTAG,
959 "received confirmation of ICE restart"
960 + newCredentials
961 + " peer_setup="
962 + setup);
963 // DTLS setup attribute needs to be rewritten to reflect current peer state
964 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
965 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
966 }
967 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
968 return isOffer;
969 } else {
970 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
971 respondWithTieBreak(jinglePacket);
972 return true;
973 }
974 } catch (final Exception exception) {
975 respondOk(jinglePacket);
976 final Throwable rootCause = Throwables.getRootCause(exception);
977 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
978 // If this happens a termination is already in progress
979 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
980 return true;
981 }
982 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
983 webRTCWrapper.close();
984 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
985 return true;
986 }
987 }
988
989 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
990 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
991 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
992 throw new IllegalStateException("Invalid peer setup");
993 }
994 return peerSetup;
995 }
996
997 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
998 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
999 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1000 }
1001 this.peerDtlsSetup = setup;
1002 }
1003
1004 private boolean applyIceRestart(
1005 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
1006 throws ExecutionException, InterruptedException {
1007 final SessionDescription sessionDescription =
1008 SessionDescription.of(restartContentMap, isResponder());
1009 final org.webrtc.SessionDescription.Type type =
1010 isOffer
1011 ? org.webrtc.SessionDescription.Type.OFFER
1012 : org.webrtc.SessionDescription.Type.ANSWER;
1013 org.webrtc.SessionDescription sdp =
1014 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1015 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1016 if (isInitiator()) {
1017 // We ignore the offer and respond with tie-break. This will clause the responder
1018 // not to apply the content map
1019 return false;
1020 }
1021 }
1022 webRTCWrapper.setRemoteDescription(sdp).get();
1023 setRemoteContentMap(restartContentMap);
1024 if (isOffer) {
1025 final SessionDescription localSessionDescription = setLocalSessionDescription();
1026 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1027 // We need to respond OK before sending any candidates
1028 respondOk(jinglePacket);
1029 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1030 } else {
1031 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1032 }
1033 return true;
1034 }
1035
1036 private void processCandidates(
1037 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1038 contents) {
1039 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1040 content : contents) {
1041 processCandidate(content);
1042 }
1043 }
1044
1045 private void processCandidate(
1046 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1047 content) {
1048 final RtpContentMap rtpContentMap = getRemoteContentMap();
1049 final List<String> indices = toIdentificationTags(rtpContentMap);
1050 final String sdpMid = content.getKey(); // aka content name
1051 final IceUdpTransportInfo transport = content.getValue().transport;
1052 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1053
1054 // TODO check that credentials remained the same
1055
1056 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1057 final String sdp;
1058 try {
1059 sdp = candidate.toSdpAttribute(credentials.ufrag);
1060 } catch (final IllegalArgumentException e) {
1061 Log.d(
1062 Config.LOGTAG,
1063 id.account.getJid().asBareJid()
1064 + ": ignoring invalid ICE candidate "
1065 + e.getMessage());
1066 continue;
1067 }
1068 final int mLineIndex = indices.indexOf(sdpMid);
1069 if (mLineIndex < 0) {
1070 Log.w(
1071 Config.LOGTAG,
1072 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1073 }
1074 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1075 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1076 this.webRTCWrapper.addIceCandidate(iceCandidate);
1077 }
1078 }
1079
1080 private RtpContentMap getRemoteContentMap() {
1081 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1082 }
1083
1084 private RtpContentMap getLocalContentMap() {
1085 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1086 }
1087
1088 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1089 final Group originalGroup = rtpContentMap.group;
1090 final List<String> identificationTags =
1091 originalGroup == null
1092 ? rtpContentMap.getNames()
1093 : originalGroup.getIdentificationTags();
1094 if (identificationTags.size() == 0) {
1095 Log.w(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1099 }
1100 return identificationTags;
1101 }
1102
1103 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1104 final Jingle jinglePacket, final boolean expectVerification) {
1105 try {
1106 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1107 } catch (final Exception e) {
1108 return Futures.immediateFailedFuture(e);
1109 }
1110 }
1111
1112 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1113 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1114 Log.d(
1115 Config.LOGTAG,
1116 "receiveRtpContentMap("
1117 + receivedContentMap.getClass().getSimpleName()
1118 + ",expectVerification="
1119 + expectVerification
1120 + ")");
1121 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1122 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1123 id.account
1124 .getAxolotlService()
1125 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1126 return Futures.transform(
1127 future,
1128 omemoVerifiedPayload -> {
1129 // TODO test if an exception here triggers a correct abort
1130 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1131 Log.d(
1132 Config.LOGTAG,
1133 id.account.getJid().asBareJid()
1134 + ": received verifiable DTLS fingerprint via "
1135 + omemoVerification);
1136 return omemoVerifiedPayload.getPayload();
1137 },
1138 MoreExecutors.directExecutor());
1139 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1140 return Futures.immediateFailedFuture(
1141 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1142 } else {
1143 return Futures.immediateFuture(receivedContentMap);
1144 }
1145 }
1146
1147 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1148 if (isInitiator()) {
1149 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1150 return;
1151 }
1152 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1153 Futures.addCallback(
1154 future,
1155 new FutureCallback<>() {
1156 @Override
1157 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1158 receiveSessionInitiate(jinglePacket, rtpContentMap);
1159 }
1160
1161 @Override
1162 public void onFailure(@NonNull final Throwable throwable) {
1163 respondOk(jinglePacket);
1164 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1165 }
1166 },
1167 MoreExecutors.directExecutor());
1168 }
1169
1170 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1171 try {
1172 contentMap.requireContentDescriptions();
1173 contentMap.requireDTLSFingerprint(true);
1174 } catch (final RuntimeException e) {
1175 Log.d(
1176 Config.LOGTAG,
1177 id.account.getJid().asBareJid() + ": improperly formatted contents",
1178 Throwables.getRootCause(e));
1179 respondOk(jinglePacket);
1180 sendSessionTerminate(Reason.of(e), e.getMessage());
1181 return;
1182 }
1183 Log.d(
1184 Config.LOGTAG,
1185 "processing session-init with " + contentMap.contents.size() + " contents");
1186 final State target;
1187 if (this.state == State.PROCEED) {
1188 Preconditions.checkState(
1189 proposedMedia != null && proposedMedia.size() > 0,
1190 "proposed media must be set when processing pre-approved session-initiate");
1191 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1192 sendSessionTerminate(
1193 Reason.SECURITY_ERROR,
1194 String.format(
1195 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1196 this.proposedMedia, contentMap.getMedia()));
1197 return;
1198 }
1199 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1200 } else {
1201 target = State.SESSION_INITIALIZED;
1202 setProposedMedia(contentMap.getMedia());
1203 }
1204 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1205 respondOk(jinglePacket);
1206 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1207 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1208 Log.d(
1209 Config.LOGTAG,
1210 id.account.getJid().asBareJid()
1211 + ": automatically accepting session-initiate");
1212 sendSessionAccept();
1213 } else {
1214 Log.d(
1215 Config.LOGTAG,
1216 id.account.getJid().asBareJid()
1217 + ": received not pre-approved session-initiate. start ringing");
1218 startRinging();
1219 }
1220 } else {
1221 Log.d(
1222 Config.LOGTAG,
1223 String.format(
1224 "%s: received session-initiate while in state %s",
1225 id.account.getJid().asBareJid(), state));
1226 terminateWithOutOfOrder(jinglePacket);
1227 }
1228 }
1229
1230 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1231 if (isResponder()) {
1232 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1233 return;
1234 }
1235 final ListenableFuture<RtpContentMap> future =
1236 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1237 Futures.addCallback(
1238 future,
1239 new FutureCallback<>() {
1240 @Override
1241 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1242 receiveSessionAccept(jinglePacket, rtpContentMap);
1243 }
1244
1245 @Override
1246 public void onFailure(@NonNull final Throwable throwable) {
1247 respondOk(jinglePacket);
1248 Log.d(
1249 Config.LOGTAG,
1250 id.account.getJid().asBareJid()
1251 + ": improperly formatted contents in session-accept",
1252 throwable);
1253 webRTCWrapper.close();
1254 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1255 }
1256 },
1257 MoreExecutors.directExecutor());
1258 }
1259
1260 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1261 try {
1262 contentMap.requireContentDescriptions();
1263 contentMap.requireDTLSFingerprint();
1264 } catch (final RuntimeException e) {
1265 respondOk(jinglePacket);
1266 Log.d(
1267 Config.LOGTAG,
1268 id.account.getJid().asBareJid()
1269 + ": improperly formatted contents in session-accept",
1270 e);
1271 webRTCWrapper.close();
1272 sendSessionTerminate(Reason.of(e), e.getMessage());
1273 return;
1274 }
1275 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1276 if (!initiatorMedia.equals(contentMap.getMedia())) {
1277 sendSessionTerminate(
1278 Reason.SECURITY_ERROR,
1279 String.format(
1280 "Your session-included included media %s but our session-initiate was %s",
1281 this.proposedMedia, contentMap.getMedia()));
1282 return;
1283 }
1284 Log.d(
1285 Config.LOGTAG,
1286 "processing session-accept with " + contentMap.contents.size() + " contents");
1287 if (transition(State.SESSION_ACCEPTED)) {
1288 respondOk(jinglePacket);
1289 receiveSessionAccept(contentMap);
1290 } else {
1291 Log.d(
1292 Config.LOGTAG,
1293 String.format(
1294 "%s: received session-accept while in state %s",
1295 id.account.getJid().asBareJid(), state));
1296 respondOk(jinglePacket);
1297 }
1298 }
1299
1300 private void receiveSessionAccept(final RtpContentMap contentMap) {
1301 this.responderRtpContentMap = contentMap;
1302 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1303 final SessionDescription sessionDescription;
1304 try {
1305 sessionDescription = SessionDescription.of(contentMap, false);
1306 } catch (final IllegalArgumentException | NullPointerException e) {
1307 Log.d(
1308 Config.LOGTAG,
1309 id.account.getJid().asBareJid()
1310 + ": unable convert offer from session-accept to SDP",
1311 e);
1312 webRTCWrapper.close();
1313 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1314 return;
1315 }
1316 final org.webrtc.SessionDescription answer =
1317 new org.webrtc.SessionDescription(
1318 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1319 try {
1320 this.webRTCWrapper.setRemoteDescription(answer).get();
1321 } catch (final Exception e) {
1322 Log.d(
1323 Config.LOGTAG,
1324 id.account.getJid().asBareJid()
1325 + ": unable to set remote description after receiving session-accept",
1326 Throwables.getRootCause(e));
1327 webRTCWrapper.close();
1328 sendSessionTerminate(
1329 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1330 return;
1331 }
1332 processCandidates(contentMap.contents.entrySet());
1333 }
1334
1335 private void sendSessionAccept() {
1336 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1337 if (rtpContentMap == null) {
1338 throw new IllegalStateException("initiator RTP Content Map has not been set");
1339 }
1340 final SessionDescription offer;
1341 try {
1342 offer = SessionDescription.of(rtpContentMap, true);
1343 } catch (final IllegalArgumentException | NullPointerException e) {
1344 Log.d(
1345 Config.LOGTAG,
1346 id.account.getJid().asBareJid()
1347 + ": unable convert offer from session-initiate to SDP",
1348 e);
1349 webRTCWrapper.close();
1350 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1351 return;
1352 }
1353 sendSessionAccept(rtpContentMap.getMedia(), offer);
1354 }
1355
1356 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1357 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1358 }
1359
1360 private synchronized void sendSessionAccept(
1361 final Set<Media> media,
1362 final SessionDescription offer,
1363 final List<PeerConnection.IceServer> iceServers) {
1364 if (isTerminated()) {
1365 Log.w(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid()
1368 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1369 return;
1370 }
1371 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1372 try {
1373 setupWebRTC(media, iceServers, !includeCandidates);
1374 } catch (final WebRTCWrapper.InitializationException e) {
1375 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1376 webRTCWrapper.close();
1377 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1378 return;
1379 }
1380 final org.webrtc.SessionDescription sdp =
1381 new org.webrtc.SessionDescription(
1382 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1383 try {
1384 this.webRTCWrapper.setRemoteDescription(sdp).get();
1385 addIceCandidatesFromBlackLog();
1386 org.webrtc.SessionDescription webRTCSessionDescription =
1387 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1388 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1389 } catch (final Exception e) {
1390 failureToAcceptSession(e);
1391 }
1392 }
1393
1394 private void failureToAcceptSession(final Throwable throwable) {
1395 if (isTerminated()) {
1396 return;
1397 }
1398 final Throwable rootCause = Throwables.getRootCause(throwable);
1399 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1400 webRTCWrapper.close();
1401 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1402 }
1403
1404 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1405 if (isTerminated()) {
1406 return;
1407 }
1408 final Throwable rootCause = Throwables.getRootCause(throwable);
1409 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1410 webRTCWrapper.close();
1411 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1412 }
1413
1414 private void addIceCandidatesFromBlackLog() {
1415 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1416 while ((foo = this.pendingIceCandidates.poll()) != null) {
1417 processCandidate(foo);
1418 Log.d(
1419 Config.LOGTAG,
1420 id.account.getJid().asBareJid() + ": added candidate from back log");
1421 }
1422 }
1423
1424 private void prepareSessionAccept(
1425 final org.webrtc.SessionDescription webRTCSessionDescription,
1426 final boolean includeCandidates) {
1427 final SessionDescription sessionDescription =
1428 SessionDescription.parse(webRTCSessionDescription.description);
1429 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1430 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1431 if (includeCandidates) {
1432 candidates = parseCandidates(sessionDescription);
1433 } else {
1434 candidates = ImmutableMultimap.of();
1435 }
1436 this.responderRtpContentMap = respondingRtpContentMap;
1437 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1438 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1439 prepareOutgoingContentMap(respondingRtpContentMap);
1440 Futures.addCallback(
1441 outgoingContentMapFuture,
1442 new FutureCallback<>() {
1443 @Override
1444 public void onSuccess(final RtpContentMap outgoingContentMap) {
1445 if (includeCandidates) {
1446 Log.d(
1447 Config.LOGTAG,
1448 "including "
1449 + candidates.size()
1450 + " candidates in session accept");
1451 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1452 } else {
1453 sendSessionAccept(outgoingContentMap);
1454 }
1455 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1456 }
1457
1458 @Override
1459 public void onFailure(@NonNull Throwable throwable) {
1460 failureToAcceptSession(throwable);
1461 }
1462 },
1463 MoreExecutors.directExecutor());
1464 }
1465
1466 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1467 if (isTerminated()) {
1468 Log.w(
1469 Config.LOGTAG,
1470 id.account.getJid().asBareJid()
1471 + ": preparing session accept was too slow. already terminated. nothing to do.");
1472 return;
1473 }
1474 transitionOrThrow(State.SESSION_ACCEPTED);
1475 final Iq sessionAccept =
1476 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1477 send(sessionAccept);
1478 }
1479
1480 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1481 final RtpContentMap rtpContentMap) {
1482 if (this.omemoVerification.hasDeviceId()) {
1483 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1484 verifiedPayloadFuture =
1485 id.account
1486 .getAxolotlService()
1487 .encrypt(
1488 rtpContentMap,
1489 id.with,
1490 omemoVerification.getDeviceId());
1491 return Futures.transform(
1492 verifiedPayloadFuture,
1493 verifiedPayload -> {
1494 omemoVerification.setOrEnsureEqual(verifiedPayload);
1495 return verifiedPayload.getPayload();
1496 },
1497 MoreExecutors.directExecutor());
1498 } else {
1499 return Futures.immediateFuture(rtpContentMap);
1500 }
1501 }
1502
1503 synchronized void deliveryMessage(
1504 final Jid from,
1505 final Element message,
1506 final String serverMessageId,
1507 final long timestamp) {
1508 Log.d(
1509 Config.LOGTAG,
1510 id.account.getJid().asBareJid()
1511 + ": delivered message to JingleRtpConnection "
1512 + message);
1513 switch (message.getName()) {
1514 case "propose" -> receivePropose(
1515 from, Propose.upgrade(message), serverMessageId, timestamp);
1516 case "proceed" -> receiveProceed(
1517 from, Proceed.upgrade(message), serverMessageId, timestamp);
1518 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1519 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1520 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1521 }
1522 }
1523
1524 void deliverFailedProceed(final String message) {
1525 Log.d(
1526 Config.LOGTAG,
1527 id.account.getJid().asBareJid()
1528 + ": receive message error for proceed message ("
1529 + Strings.nullToEmpty(message)
1530 + ")");
1531 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1532 webRTCWrapper.close();
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1536 this.finish();
1537 }
1538 }
1539
1540 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1541 final boolean originatedFromMyself =
1542 from.asBareJid().equals(id.account.getJid().asBareJid());
1543 if (originatedFromMyself) {
1544 if (transition(State.ACCEPTED)) {
1545 acceptedOnOtherDevice(serverMsgId, timestamp);
1546 } else {
1547 Log.d(
1548 Config.LOGTAG,
1549 id.account.getJid().asBareJid()
1550 + ": unable to transition to accept because already in state="
1551 + this.state);
1552 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1553 }
1554 } else {
1555 Log.d(
1556 Config.LOGTAG,
1557 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1558 }
1559 }
1560
1561 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1562 if (serverMsgId != null) {
1563 this.message.setServerMsgId(serverMsgId);
1564 }
1565 this.message.setTime(timestamp);
1566 this.message.setCarbon(true); // indicate that call was accepted on other device
1567 this.writeLogMessageSuccess(0);
1568 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1569 this.finish();
1570 }
1571
1572 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1573 final boolean originatedFromMyself =
1574 from.asBareJid().equals(id.account.getJid().asBareJid());
1575 // reject from another one of my clients
1576 if (originatedFromMyself) {
1577 receiveRejectFromMyself(serverMsgId, timestamp);
1578 } else if (isInitiator()) {
1579 if (from.equals(id.with)) {
1580 receiveRejectFromResponder();
1581 } else {
1582 Log.d(
1583 Config.LOGTAG,
1584 id.account.getJid()
1585 + ": ignoring reject from "
1586 + from
1587 + " for session with "
1588 + id.with);
1589 }
1590 } else {
1591 Log.d(
1592 Config.LOGTAG,
1593 id.account.getJid()
1594 + ": ignoring reject from "
1595 + from
1596 + " for session with "
1597 + id.with);
1598 }
1599 }
1600
1601 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1602 if (transition(State.REJECTED)) {
1603 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1604 this.finish();
1605 if (serverMsgId != null) {
1606 this.message.setServerMsgId(serverMsgId);
1607 }
1608 this.message.setTime(timestamp);
1609 this.message.setCarbon(true); // indicate that call was rejected on other device
1610 writeLogMessageMissed();
1611 } else {
1612 Log.d(
1613 Config.LOGTAG,
1614 "not able to transition into REJECTED because already in " + this.state);
1615 }
1616 }
1617
1618 private void receiveRejectFromResponder() {
1619 if (isInState(State.PROCEED)) {
1620 Log.d(
1621 Config.LOGTAG,
1622 id.account.getJid()
1623 + ": received reject while still in proceed. callee reconsidered");
1624 closeTransitionLogFinish(State.REJECTED_RACED);
1625 return;
1626 }
1627 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1628 Log.d(
1629 Config.LOGTAG,
1630 id.account.getJid()
1631 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1632 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1633 return;
1634 }
1635 Log.d(
1636 Config.LOGTAG,
1637 id.account.getJid()
1638 + ": ignoring reject from responder because already in state "
1639 + this.state);
1640 }
1641
1642 private void receivePropose(
1643 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1644 final boolean originatedFromMyself =
1645 from.asBareJid().equals(id.account.getJid().asBareJid());
1646 if (originatedFromMyself) {
1647 Log.d(
1648 Config.LOGTAG,
1649 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1650 } else if (transition(
1651 State.PROPOSED,
1652 () -> {
1653 final Collection<RtpDescription> descriptions =
1654 Collections2.transform(
1655 Collections2.filter(
1656 propose.getDescriptions(),
1657 d -> d instanceof RtpDescription),
1658 input -> (RtpDescription) input);
1659 final Collection<Media> media =
1660 Collections2.transform(descriptions, RtpDescription::getMedia);
1661 Preconditions.checkState(
1662 !media.contains(Media.UNKNOWN),
1663 "RTP descriptions contain unknown media");
1664 Log.d(
1665 Config.LOGTAG,
1666 id.account.getJid().asBareJid()
1667 + ": received session proposal from "
1668 + from
1669 + " for "
1670 + media);
1671 this.setProposedMedia(Sets.newHashSet(media));
1672 })) {
1673 if (serverMsgId != null) {
1674 this.message.setServerMsgId(serverMsgId);
1675 }
1676 this.message.setTime(timestamp);
1677 startRinging();
1678 // in environments where we always use discovery timeouts we always want to respond with
1679 // 'ringing'
1680 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1681 || (xmppConnectionService.confirmMessages()
1682 && id.getContact().showInContactList())) {
1683 sendJingleMessage("ringing");
1684 }
1685 } else {
1686 Log.d(
1687 Config.LOGTAG,
1688 id.account.getJid()
1689 + ": ignoring session proposal because already in "
1690 + state);
1691 }
1692 }
1693
1694 private void startRinging() {
1695 this.callIntegration.setRinging();
1696 Log.d(
1697 Config.LOGTAG,
1698 id.account.getJid().asBareJid()
1699 + ": received call from "
1700 + id.with
1701 + ". start ringing");
1702 ringingTimeoutFuture =
1703 jingleConnectionManager.schedule(
1704 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1705 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1706 return;
1707 }
1708 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1709 }
1710
1711 private synchronized void ringingTimeout() {
1712 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1713 switch (this.state) {
1714 case PROPOSED -> {
1715 message.markUnread();
1716 rejectCallFromProposed();
1717 }
1718 case SESSION_INITIALIZED -> {
1719 message.markUnread();
1720 rejectCallFromSessionInitiate();
1721 }
1722 }
1723 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1724 }
1725
1726 private void cancelRingingTimeout() {
1727 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1728 if (future != null && !future.isCancelled()) {
1729 future.cancel(false);
1730 }
1731 }
1732
1733 private void receiveProceed(
1734 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1735 final Set<Media> media =
1736 Preconditions.checkNotNull(
1737 this.proposedMedia, "Proposed media has to be set before handling proceed");
1738 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1739 if (from.equals(id.with)) {
1740 if (isInitiator()) {
1741 if (transition(State.PROCEED)) {
1742 if (serverMsgId != null) {
1743 this.message.setServerMsgId(serverMsgId);
1744 }
1745 this.message.setTime(timestamp);
1746 final Integer remoteDeviceId = proceed.getDeviceId();
1747 if (isOmemoEnabled()) {
1748 this.omemoVerification.setDeviceId(remoteDeviceId);
1749 } else {
1750 if (remoteDeviceId != null) {
1751 Log.d(
1752 Config.LOGTAG,
1753 id.account.getJid().asBareJid()
1754 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1755 }
1756 this.omemoVerification.setDeviceId(null);
1757 }
1758 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1759 } else {
1760 Log.d(
1761 Config.LOGTAG,
1762 String.format(
1763 "%s: ignoring proceed because already in %s",
1764 id.account.getJid().asBareJid(), this.state));
1765 }
1766 } else {
1767 Log.d(
1768 Config.LOGTAG,
1769 String.format(
1770 "%s: ignoring proceed because we were not initializing",
1771 id.account.getJid().asBareJid()));
1772 }
1773 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1774 if (transition(State.ACCEPTED)) {
1775 Log.d(
1776 Config.LOGTAG,
1777 id.account.getJid().asBareJid()
1778 + ": moved session with "
1779 + id.with
1780 + " into state accepted after received carbon copied proceed");
1781 acceptedOnOtherDevice(serverMsgId, timestamp);
1782 }
1783 } else {
1784 Log.d(
1785 Config.LOGTAG,
1786 String.format(
1787 "%s: ignoring proceed from %s. was expected from %s",
1788 id.account.getJid().asBareJid(), from, id.with));
1789 }
1790 }
1791
1792 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1793 if (from.equals(id.with)) {
1794 final State target =
1795 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1796 if (transition(target)) {
1797 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1798 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1799 Log.d(
1800 Config.LOGTAG,
1801 id.account.getJid().asBareJid()
1802 + ": session with "
1803 + id.with
1804 + " has been retracted (serverMsgId="
1805 + serverMsgId
1806 + ")");
1807 if (serverMsgId != null) {
1808 this.message.setServerMsgId(serverMsgId);
1809 }
1810 this.message.setTime(timestamp);
1811 if (target == State.RETRACTED) {
1812 this.message.markUnread();
1813 }
1814 writeLogMessageMissed();
1815 finish();
1816 } else {
1817 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1818 }
1819 } else {
1820 // TODO parse retract from self
1821 Log.d(
1822 Config.LOGTAG,
1823 id.account.getJid().asBareJid()
1824 + ": received retract from "
1825 + from
1826 + ". expected retract from"
1827 + id.with
1828 + ". ignoring");
1829 }
1830 }
1831
1832 public void sendSessionInitiate() {
1833 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1834 }
1835
1836 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1837 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1838 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1839 }
1840
1841 private synchronized void sendSessionInitiate(
1842 final Set<Media> media,
1843 final State targetState,
1844 final List<PeerConnection.IceServer> iceServers) {
1845 if (isTerminated()) {
1846 Log.w(
1847 Config.LOGTAG,
1848 id.account.getJid().asBareJid()
1849 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1850 return;
1851 }
1852 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1853 try {
1854 setupWebRTC(media, iceServers, !includeCandidates);
1855 } catch (final WebRTCWrapper.InitializationException e) {
1856 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1857 webRTCWrapper.close();
1858 sendRetract(Reason.ofThrowable(e));
1859 return;
1860 }
1861 try {
1862 org.webrtc.SessionDescription webRTCSessionDescription =
1863 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1864 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1865 } catch (final Exception e) {
1866 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1867 // exceptions
1868 failureToInitiateSession(e, targetState);
1869 }
1870 }
1871
1872 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1873 if (isTerminated()) {
1874 return;
1875 }
1876 Log.d(
1877 Config.LOGTAG,
1878 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1879 Throwables.getRootCause(throwable));
1880 webRTCWrapper.close();
1881 final Reason reason = Reason.ofThrowable(throwable);
1882 if (isInState(targetState)) {
1883 sendSessionTerminate(reason, throwable.getMessage());
1884 } else {
1885 sendRetract(reason);
1886 }
1887 }
1888
1889 private void sendRetract(final Reason reason) {
1890 // TODO embed reason into retract
1891 sendJingleMessage("retract", id.with.asBareJid());
1892 transitionOrThrow(reasonToState(reason));
1893 this.finish();
1894 }
1895
1896 private void prepareSessionInitiate(
1897 final org.webrtc.SessionDescription webRTCSessionDescription,
1898 final boolean includeCandidates,
1899 final State targetState) {
1900 final SessionDescription sessionDescription =
1901 SessionDescription.parse(webRTCSessionDescription.description);
1902 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1903 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1904 if (includeCandidates) {
1905 candidates = parseCandidates(sessionDescription);
1906 } else {
1907 candidates = ImmutableMultimap.of();
1908 }
1909 this.initiatorRtpContentMap = rtpContentMap;
1910 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1911 encryptSessionInitiate(rtpContentMap);
1912 Futures.addCallback(
1913 outgoingContentMapFuture,
1914 new FutureCallback<>() {
1915 @Override
1916 public void onSuccess(final RtpContentMap outgoingContentMap) {
1917 if (includeCandidates) {
1918 Log.d(
1919 Config.LOGTAG,
1920 "including "
1921 + candidates.size()
1922 + " candidates in session initiate");
1923 sendSessionInitiate(
1924 outgoingContentMap.withCandidates(candidates), targetState);
1925 } else {
1926 sendSessionInitiate(outgoingContentMap, targetState);
1927 }
1928 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1929 }
1930
1931 @Override
1932 public void onFailure(@NonNull final Throwable throwable) {
1933 failureToInitiateSession(throwable, targetState);
1934 }
1935 },
1936 MoreExecutors.directExecutor());
1937 }
1938
1939 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1940 if (isTerminated()) {
1941 Log.w(
1942 Config.LOGTAG,
1943 id.account.getJid().asBareJid()
1944 + ": preparing session was too slow. already terminated. nothing to do.");
1945 return;
1946 }
1947 this.transitionOrThrow(targetState);
1948 final Iq sessionInitiate =
1949 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1950 send(sessionInitiate);
1951 }
1952
1953 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1954 final RtpContentMap rtpContentMap) {
1955 if (this.omemoVerification.hasDeviceId()) {
1956 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1957 verifiedPayloadFuture =
1958 id.account
1959 .getAxolotlService()
1960 .encrypt(
1961 rtpContentMap,
1962 id.with,
1963 omemoVerification.getDeviceId());
1964 final ListenableFuture<RtpContentMap> future =
1965 Futures.transform(
1966 verifiedPayloadFuture,
1967 verifiedPayload -> {
1968 omemoVerification.setSessionFingerprint(
1969 verifiedPayload.getFingerprint());
1970 return verifiedPayload.getPayload();
1971 },
1972 MoreExecutors.directExecutor());
1973 if (Config.REQUIRE_RTP_VERIFICATION) {
1974 return future;
1975 }
1976 return Futures.catching(
1977 future,
1978 CryptoFailedException.class,
1979 e -> {
1980 Log.w(
1981 Config.LOGTAG,
1982 id.account.getJid().asBareJid()
1983 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1984 e);
1985 return rtpContentMap;
1986 },
1987 MoreExecutors.directExecutor());
1988 } else {
1989 return Futures.immediateFuture(rtpContentMap);
1990 }
1991 }
1992
1993 protected void sendSessionTerminate(final Reason reason) {
1994 sendSessionTerminate(reason, null);
1995 }
1996
1997 protected void sendSessionTerminate(final Reason reason, final String text) {
1998 sendSessionTerminate(reason, text, this::writeLogMessage);
1999 sendJingleMessageFinish(reason);
2000 }
2001
2002 private void sendTransportInfo(
2003 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2004 final RtpContentMap transportInfo;
2005 try {
2006 final RtpContentMap rtpContentMap =
2007 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2008 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2009 } catch (final Exception e) {
2010 Log.d(
2011 Config.LOGTAG,
2012 id.account.getJid().asBareJid()
2013 + ": unable to prepare transport-info from candidate for content="
2014 + contentName);
2015 return;
2016 }
2017 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2018 send(iq);
2019 }
2020
2021 public RtpEndUserState getEndUserState() {
2022 switch (this.state) {
2023 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2024 if (isInitiator()) {
2025 return RtpEndUserState.RINGING;
2026 } else {
2027 return RtpEndUserState.INCOMING_CALL;
2028 }
2029 }
2030 case PROCEED -> {
2031 if (isInitiator()) {
2032 return RtpEndUserState.RINGING;
2033 } else {
2034 return RtpEndUserState.ACCEPTING_CALL;
2035 }
2036 }
2037 case SESSION_INITIALIZED_PRE_APPROVED -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.RINGING;
2040 } else {
2041 return RtpEndUserState.CONNECTING;
2042 }
2043 }
2044 case SESSION_ACCEPTED -> {
2045 final ContentAddition ca = getPendingContentAddition();
2046 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2047 return RtpEndUserState.INCOMING_CONTENT_ADD;
2048 }
2049 return getPeerConnectionStateAsEndUserState();
2050 }
2051 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2052 if (isInitiator()) {
2053 return RtpEndUserState.DECLINED_OR_BUSY;
2054 } else {
2055 return RtpEndUserState.ENDED;
2056 }
2057 }
2058 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2059 return RtpEndUserState.ENDED;
2060 }
2061 case RETRACTED_RACED -> {
2062 if (isInitiator()) {
2063 return RtpEndUserState.ENDED;
2064 } else {
2065 return RtpEndUserState.RETRACTED;
2066 }
2067 }
2068 case TERMINATED_CONNECTIVITY_ERROR -> {
2069 return zeroDuration()
2070 ? RtpEndUserState.CONNECTIVITY_ERROR
2071 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2072 }
2073 case TERMINATED_APPLICATION_FAILURE -> {
2074 return RtpEndUserState.APPLICATION_ERROR;
2075 }
2076 case TERMINATED_SECURITY_ERROR -> {
2077 return RtpEndUserState.SECURITY_ERROR;
2078 }
2079 }
2080 throw new IllegalStateException(
2081 String.format("%s has no equivalent EndUserState", this.state));
2082 }
2083
2084 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2085 final PeerConnection.PeerConnectionState state;
2086 try {
2087 state = webRTCWrapper.getState();
2088 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2089 // We usually close the WebRTCWrapper *before* transitioning so we might still
2090 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2091 return RtpEndUserState.ENDING_CALL;
2092 }
2093 return switch (state) {
2094 case CONNECTED -> RtpEndUserState.CONNECTED;
2095 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2096 case CLOSED -> RtpEndUserState.ENDING_CALL;
2097 default -> zeroDuration()
2098 ? RtpEndUserState.CONNECTIVITY_ERROR
2099 : RtpEndUserState.RECONNECTING;
2100 };
2101 }
2102
2103 private boolean isPeerConnectionConnected() {
2104 try {
2105 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2106 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2107 return false;
2108 }
2109 }
2110
2111 private void updateCallIntegrationState() {
2112 switch (this.state) {
2113 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2114 if (isInitiator()) {
2115 this.callIntegration.setDialing();
2116 } else {
2117 this.callIntegration.setRinging();
2118 }
2119 }
2120 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2121 if (isInitiator()) {
2122 this.callIntegration.setDialing();
2123 } else {
2124 this.callIntegration.setInitialized();
2125 }
2126 }
2127 case SESSION_ACCEPTED -> {
2128 if (isPeerConnectionConnected()) {
2129 this.callIntegration.setActive();
2130 } else {
2131 this.callIntegration.setInitialized();
2132 }
2133 }
2134 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2135 if (isInitiator()) {
2136 this.callIntegration.busy();
2137 } else {
2138 this.callIntegration.rejected();
2139 }
2140 }
2141 case TERMINATED_SUCCESS -> this.callIntegration.success();
2142 case ACCEPTED -> this.callIntegration.accepted();
2143 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2144 .retracted();
2145 case TERMINATED_CONNECTIVITY_ERROR,
2146 TERMINATED_APPLICATION_FAILURE,
2147 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2148 default -> throw new IllegalStateException(
2149 String.format("%s is not handled", this.state));
2150 }
2151 }
2152
2153 public ContentAddition getPendingContentAddition() {
2154 final RtpContentMap in = this.incomingContentAdd;
2155 final RtpContentMap out = this.outgoingContentAdd;
2156 if (out != null) {
2157 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2158 } else if (in != null) {
2159 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2160 } else {
2161 return null;
2162 }
2163 }
2164
2165 public Set<Media> getMedia() {
2166 final State current = getState();
2167 if (current == State.NULL) {
2168 if (isInitiator()) {
2169 return Preconditions.checkNotNull(
2170 this.proposedMedia, "RTP connection has not been initialized properly");
2171 }
2172 throw new IllegalStateException("RTP connection has not been initialized yet");
2173 }
2174 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2175 return Preconditions.checkNotNull(
2176 this.proposedMedia, "RTP connection has not been initialized properly");
2177 }
2178 final RtpContentMap localContentMap = getLocalContentMap();
2179 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2180 if (localContentMap != null) {
2181 return localContentMap.getMedia();
2182 } else if (initiatorContentMap != null) {
2183 return initiatorContentMap.getMedia();
2184 } else if (isTerminated()) {
2185 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2186 } else {
2187 return Preconditions.checkNotNull(
2188 this.proposedMedia, "RTP connection has not been initialized properly");
2189 }
2190 }
2191
2192 public boolean isVerified() {
2193 final String fingerprint = this.omemoVerification.getFingerprint();
2194 if (fingerprint == null) {
2195 return false;
2196 }
2197 final FingerprintStatus status =
2198 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2199 return status != null && status.isVerified();
2200 }
2201
2202 public boolean addMedia(final Media media) {
2203 final Set<Media> currentMedia = getMedia();
2204 if (currentMedia.contains(media)) {
2205 throw new IllegalStateException(String.format("%s has already been proposed", media));
2206 }
2207 // TODO add state protection - can only add while ACCEPTED or so
2208 Log.d(Config.LOGTAG, "adding media: " + media);
2209 return webRTCWrapper.addTrack(media);
2210 }
2211
2212 public synchronized void acceptCall() {
2213 switch (this.state) {
2214 case PROPOSED -> {
2215 cancelRingingTimeout();
2216 acceptCallFromProposed();
2217 }
2218 case SESSION_INITIALIZED -> {
2219 cancelRingingTimeout();
2220 acceptCallFromSessionInitialized();
2221 }
2222 case ACCEPTED -> Log.w(
2223 Config.LOGTAG,
2224 id.account.getJid().asBareJid()
2225 + ": the call has already been accepted with another client. UI was just lagging behind");
2226 case PROCEED, SESSION_ACCEPTED -> Log.w(
2227 Config.LOGTAG,
2228 id.account.getJid().asBareJid()
2229 + ": the call has already been accepted. user probably double tapped the UI");
2230 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2231 }
2232 }
2233
2234 public synchronized void rejectCall() {
2235 if (isTerminated()) {
2236 Log.w(
2237 Config.LOGTAG,
2238 id.account.getJid().asBareJid()
2239 + ": received rejectCall() when session has already been terminated. nothing to do");
2240 return;
2241 }
2242 switch (this.state) {
2243 case PROPOSED -> rejectCallFromProposed();
2244 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2245 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2246 }
2247 }
2248
2249 public synchronized void integrationFailure() {
2250 final var state = getState();
2251 if (state == State.PROPOSED) {
2252 Log.e(
2253 Config.LOGTAG,
2254 id.account.getJid().asBareJid()
2255 + ": failed call integration in state proposed");
2256 rejectCallFromProposed();
2257 } else if (state == State.SESSION_INITIALIZED) {
2258 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2259 this.webRTCWrapper.close();
2260 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2261 } else {
2262 throw new IllegalStateException(
2263 String.format("Can not fail integration in state %s", state));
2264 }
2265 }
2266
2267 public synchronized void endCall() {
2268 if (isTerminated()) {
2269 Log.w(
2270 Config.LOGTAG,
2271 id.account.getJid().asBareJid()
2272 + ": received endCall() when session has already been terminated. nothing to do");
2273 return;
2274 }
2275 if (isInState(State.PROPOSED) && isResponder()) {
2276 rejectCallFromProposed();
2277 return;
2278 }
2279 if (isInState(State.PROCEED)) {
2280 if (isInitiator()) {
2281 retractFromProceed();
2282 } else {
2283 rejectCallFromProceed();
2284 }
2285 return;
2286 }
2287 if (isInitiator()
2288 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2289 this.webRTCWrapper.close();
2290 sendSessionTerminate(Reason.CANCEL);
2291 return;
2292 }
2293 if (isInState(State.SESSION_INITIALIZED)) {
2294 rejectCallFromSessionInitiate();
2295 return;
2296 }
2297 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2298 this.webRTCWrapper.close();
2299 sendSessionTerminate(Reason.SUCCESS);
2300 return;
2301 }
2302 if (isInState(
2303 State.TERMINATED_APPLICATION_FAILURE,
2304 State.TERMINATED_CONNECTIVITY_ERROR,
2305 State.TERMINATED_DECLINED_OR_BUSY)) {
2306 Log.d(
2307 Config.LOGTAG,
2308 "ignoring request to end call because already in state " + this.state);
2309 return;
2310 }
2311 throw new IllegalStateException(
2312 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2313 }
2314
2315 private void retractFromProceed() {
2316 Log.d(Config.LOGTAG, "retract from proceed");
2317 this.sendJingleMessage("retract");
2318 closeTransitionLogFinish(State.RETRACTED_RACED);
2319 }
2320
2321 private void closeTransitionLogFinish(final State state) {
2322 this.webRTCWrapper.close();
2323 transitionOrThrow(state);
2324 writeLogMessage(state);
2325 finish();
2326 }
2327
2328 private void setupWebRTC(
2329 final Set<Media> media,
2330 final List<PeerConnection.IceServer> iceServers,
2331 final boolean trickle)
2332 throws WebRTCWrapper.InitializationException {
2333 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2334 this.webRTCWrapper.setup(this.xmppConnectionService);
2335 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2336 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2337 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2338 }
2339
2340 private void acceptCallFromProposed() {
2341 transitionOrThrow(State.PROCEED);
2342 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2343 this.callIntegration.startAudioRouting();
2344 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2345 this.sendJingleMessage("proceed");
2346 }
2347
2348 private void rejectCallFromProposed() {
2349 transitionOrThrow(State.REJECTED);
2350 writeLogMessageMissed();
2351 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2352 this.sendJingleMessage("reject");
2353 finish();
2354 }
2355
2356 private void rejectCallFromProceed() {
2357 this.sendJingleMessage("reject");
2358 closeTransitionLogFinish(State.REJECTED_RACED);
2359 }
2360
2361 private void rejectCallFromSessionInitiate() {
2362 webRTCWrapper.close();
2363 sendSessionTerminate(Reason.DECLINE);
2364 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2365 }
2366
2367 private void sendJingleMessage(final String action) {
2368 sendJingleMessage(action, id.with);
2369 }
2370
2371 private void sendJingleMessage(final String action, final Jid to) {
2372 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2373 messagePacket.setType(
2374 im.conversations.android.xmpp.model.stanza.Message.Type
2375 .CHAT); // we want to carbon copy those
2376 messagePacket.setTo(to);
2377 final Element intent =
2378 messagePacket
2379 .addChild(action, Namespace.JINGLE_MESSAGE)
2380 .setAttribute("id", id.sessionId);
2381 if ("proceed".equals(action)) {
2382 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2383 if (isOmemoEnabled()) {
2384 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2385 final Element device =
2386 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2387 device.setAttribute("id", deviceId);
2388 }
2389 }
2390 messagePacket.addChild("store", "urn:xmpp:hints");
2391 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2392 }
2393
2394 private void sendJingleMessageFinish(final Reason reason) {
2395 final var account = id.getAccount();
2396 final var messagePacket =
2397 xmppConnectionService
2398 .getMessageGenerator()
2399 .sessionFinish(id.with, id.sessionId, reason);
2400 xmppConnectionService.sendMessagePacket(account, messagePacket);
2401 }
2402
2403 private boolean isOmemoEnabled() {
2404 final Conversational conversational = message.getConversation();
2405 if (conversational instanceof Conversation) {
2406 return ((Conversation) conversational).getNextEncryption()
2407 == Message.ENCRYPTION_AXOLOTL;
2408 }
2409 return false;
2410 }
2411
2412 private void acceptCallFromSessionInitialized() {
2413 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2414 this.callIntegration.startAudioRouting();
2415 sendSessionAccept();
2416 }
2417
2418 @Override
2419 protected synchronized boolean transition(final State target, final Runnable runnable) {
2420 if (super.transition(target, runnable)) {
2421 updateEndUserState();
2422 updateOngoingCallNotification();
2423 return true;
2424 } else {
2425 return false;
2426 }
2427 }
2428
2429 @Override
2430 public void onIceCandidate(final IceCandidate iceCandidate) {
2431 final RtpContentMap rtpContentMap =
2432 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2433 final IceUdpTransportInfo.Credentials credentials;
2434 try {
2435 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2436 } catch (final IllegalArgumentException e) {
2437 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2438 return;
2439 }
2440 final String uFrag = credentials.ufrag;
2441 final IceUdpTransportInfo.Candidate candidate =
2442 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2443 if (candidate == null) {
2444 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2445 return;
2446 }
2447 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2448 sendTransportInfo(iceCandidate.sdpMid, candidate);
2449 }
2450
2451 @Override
2452 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2453 Log.d(
2454 Config.LOGTAG,
2455 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2456 this.stateHistory.add(newState);
2457 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2458 this.sessionDuration.start();
2459 updateOngoingCallNotification();
2460 } else if (this.sessionDuration.isRunning()) {
2461 this.sessionDuration.stop();
2462 updateOngoingCallNotification();
2463 }
2464
2465 final boolean neverConnected =
2466 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2467
2468 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2469 if (neverConnected) {
2470 if (isTerminated()) {
2471 Log.d(
2472 Config.LOGTAG,
2473 id.account.getJid().asBareJid()
2474 + ": not sending session-terminate after connectivity error because session is already in state "
2475 + this.state);
2476 return;
2477 }
2478 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2479 return;
2480 } else {
2481 this.restartIce();
2482 }
2483 }
2484 updateEndUserState();
2485 }
2486
2487 private void restartIce() {
2488 this.stateHistory.clear();
2489 this.webRTCWrapper.restartIceAsync();
2490 }
2491
2492 @Override
2493 public void onRenegotiationNeeded() {
2494 this.webRTCWrapper.execute(this::renegotiate);
2495 }
2496
2497 private synchronized void renegotiate() {
2498 final SessionDescription sessionDescription;
2499 try {
2500 sessionDescription = setLocalSessionDescription();
2501 } catch (final Exception e) {
2502 final Throwable cause = Throwables.getRootCause(e);
2503 webRTCWrapper.close();
2504 if (isTerminated()) {
2505 Log.d(
2506 Config.LOGTAG,
2507 "failed to renegotiate. session was already terminated",
2508 cause);
2509 return;
2510 }
2511 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2512 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2513 return;
2514 }
2515 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2516 final RtpContentMap currentContentMap = getLocalContentMap();
2517 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2518 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2519
2520 Log.d(
2521 Config.LOGTAG,
2522 id.getAccount().getJid().asBareJid()
2523 + ": renegotiate. iceRestart="
2524 + iceRestart
2525 + " content id diff="
2526 + diff);
2527
2528 if (diff.hasModifications() && iceRestart) {
2529 webRTCWrapper.close();
2530 sendSessionTerminate(
2531 Reason.FAILED_APPLICATION,
2532 "WebRTC unexpectedly tried to modify content and transport at once");
2533 return;
2534 }
2535
2536 if (iceRestart) {
2537 initiateIceRestart(rtpContentMap);
2538 return;
2539 } else if (diff.isEmpty()) {
2540 Log.d(
2541 Config.LOGTAG,
2542 "renegotiation. nothing to do. SignalingState="
2543 + this.webRTCWrapper.getSignalingState());
2544 }
2545
2546 if (diff.added.isEmpty()) {
2547 return;
2548 }
2549 modifyLocalContentMap(rtpContentMap);
2550 sendContentAdd(rtpContentMap, diff.added);
2551 }
2552
2553 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2554 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2555 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2556 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2557 iq.setTo(id.with);
2558 xmppConnectionService.sendIqPacket(
2559 id.account,
2560 iq,
2561 (response) -> {
2562 if (response.getType() == Iq.Type.RESULT) {
2563 Log.d(Config.LOGTAG, "received success to our ice restart");
2564 setLocalContentMap(rtpContentMap);
2565 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2566 return;
2567 }
2568 if (response.getType() == Iq.Type.ERROR) {
2569 if (isTieBreak(response)) {
2570 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2571 return;
2572 }
2573 handleIqErrorResponse(response);
2574 }
2575 if (response.getType() == Iq.Type.TIMEOUT) {
2576 handleIqTimeoutResponse(response);
2577 }
2578 });
2579 }
2580
2581 private boolean isTieBreak(final Iq response) {
2582 final Element error = response.findChild("error");
2583 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2584 }
2585
2586 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2587 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2588 this.outgoingContentAdd = contentAdd;
2589 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2590 prepareOutgoingContentMap(contentAdd);
2591 Futures.addCallback(
2592 outgoingContentMapFuture,
2593 new FutureCallback<>() {
2594 @Override
2595 public void onSuccess(final RtpContentMap outgoingContentMap) {
2596 sendContentAdd(outgoingContentMap);
2597 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2598 }
2599
2600 @Override
2601 public void onFailure(@NonNull Throwable throwable) {
2602 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2603 }
2604 },
2605 MoreExecutors.directExecutor());
2606 }
2607
2608 private void sendContentAdd(final RtpContentMap contentAdd) {
2609
2610 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2611 iq.setTo(id.with);
2612 xmppConnectionService.sendIqPacket(
2613 id.account,
2614 iq,
2615 (response) -> {
2616 if (response.getType() == Iq.Type.RESULT) {
2617 Log.d(
2618 Config.LOGTAG,
2619 id.getAccount().getJid().asBareJid()
2620 + ": received ACK to our content-add");
2621 return;
2622 }
2623 if (response.getType() == Iq.Type.ERROR) {
2624 if (isTieBreak(response)) {
2625 this.outgoingContentAdd = null;
2626 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2627 return;
2628 }
2629 handleIqErrorResponse(response);
2630 }
2631 if (response.getType() == Iq.Type.TIMEOUT) {
2632 handleIqTimeoutResponse(response);
2633 }
2634 });
2635 }
2636
2637 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2638 if (isInitiator()) {
2639 this.initiatorRtpContentMap = rtpContentMap;
2640 } else {
2641 this.responderRtpContentMap = rtpContentMap;
2642 }
2643 }
2644
2645 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2646 if (isInitiator()) {
2647 this.responderRtpContentMap = rtpContentMap;
2648 } else {
2649 this.initiatorRtpContentMap = rtpContentMap;
2650 }
2651 }
2652
2653 // this method is to be used for content map modifications that modify media
2654 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2655 final RtpContentMap activeContents = rtpContentMap.activeContents();
2656 setLocalContentMap(activeContents);
2657 this.callIntegration.setAudioDeviceWhenAvailable(
2658 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2659 updateEndUserState();
2660 }
2661
2662 private SessionDescription setLocalSessionDescription()
2663 throws ExecutionException, InterruptedException {
2664 final org.webrtc.SessionDescription sessionDescription =
2665 this.webRTCWrapper.setLocalDescription(false).get();
2666 return SessionDescription.parse(sessionDescription.description);
2667 }
2668
2669 private void closeWebRTCSessionAfterFailedConnection() {
2670 this.webRTCWrapper.close();
2671 synchronized (this) {
2672 if (isTerminated()) {
2673 Log.d(
2674 Config.LOGTAG,
2675 id.account.getJid().asBareJid()
2676 + ": no need to send session-terminate after failed connection. Other party already did");
2677 return;
2678 }
2679 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2680 }
2681 }
2682
2683 public boolean zeroDuration() {
2684 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2685 }
2686
2687 public long getCallDuration() {
2688 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2689 }
2690
2691 @Override
2692 public CallIntegration getCallIntegration() {
2693 return this.callIntegration;
2694 }
2695
2696 public boolean isMicrophoneEnabled() {
2697 return webRTCWrapper.isMicrophoneEnabled();
2698 }
2699
2700 public boolean setMicrophoneEnabled(final boolean enabled) {
2701 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2702 }
2703
2704 public boolean isVideoEnabled() {
2705 return webRTCWrapper.isVideoEnabled();
2706 }
2707
2708 public void setVideoEnabled(final boolean enabled) {
2709 webRTCWrapper.setVideoEnabled(enabled);
2710 }
2711
2712 public boolean isCameraSwitchable() {
2713 return webRTCWrapper.isCameraSwitchable();
2714 }
2715
2716 public boolean isFrontCamera() {
2717 return webRTCWrapper.isFrontCamera();
2718 }
2719
2720 public ListenableFuture<Boolean> switchCamera() {
2721 return webRTCWrapper.switchCamera();
2722 }
2723
2724 @Override
2725 public synchronized void onCallIntegrationShowIncomingCallUi() {
2726 if (isTerminated()) {
2727 // there might be race conditions with the call integration service invoking this
2728 // callback when the rtp session has already ended.
2729 Log.w(
2730 Config.LOGTAG,
2731 "CallIntegration requested incoming call UI but session was already terminated");
2732 return;
2733 }
2734 // TODO apparently this can be called too early as well?
2735 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2736 }
2737
2738 @Override
2739 public void onCallIntegrationDisconnect() {
2740 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2741 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2742 rejectCall();
2743 } else {
2744 endCall();
2745 }
2746 }
2747
2748 @Override
2749 public void onCallIntegrationReject() {
2750 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2751 try {
2752 rejectCall();
2753 } catch (final IllegalStateException e) {
2754 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2755 }
2756 }
2757
2758 @Override
2759 public void onCallIntegrationAnswer() {
2760 // we need to start the UI to a) show it and b) be able to ask for permissions
2761 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2762 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2763 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2764 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2765 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2766 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2767 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2768 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2769 xmppConnectionService.startActivity(intent);
2770 }
2771
2772 @Override
2773 public void onCallIntegrationSilence() {
2774 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2775 }
2776
2777 @Override
2778 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2779 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2780 // that was previous disabled by the user. A proper implementation would probably be to
2781 // track user choice and enable the microphone with a userEnabled() &&
2782 // callIntegration.isMicrophoneEnabled() condition
2783 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2784 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2785 }
2786
2787 @Override
2788 public void onAudioDeviceChanged(
2789 final CallIntegration.AudioDevice selectedAudioDevice,
2790 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2791 Log.d(
2792 Config.LOGTAG,
2793 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2794 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2795 selectedAudioDevice, availableAudioDevices);
2796 }
2797
2798 private void updateEndUserState() {
2799 final RtpEndUserState endUserState = getEndUserState();
2800 this.updateCallIntegrationState();
2801 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2802 id.account, id.with, id.sessionId, endUserState);
2803 }
2804
2805 private void updateOngoingCallNotification() {
2806 final State state = this.state;
2807 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2808 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2809 && isResponder()) {
2810 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2811 xmppConnectionService.removeOngoingCall();
2812 return;
2813 }
2814 final boolean reconnecting;
2815 if (state == State.SESSION_ACCEPTED) {
2816 reconnecting =
2817 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2818 } else {
2819 reconnecting = false;
2820 }
2821 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2822 } else {
2823 xmppConnectionService.removeOngoingCall();
2824 }
2825 }
2826
2827 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2828 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2829 final Iq request = new Iq(Iq.Type.GET);
2830 request.setTo(id.account.getDomain());
2831 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2832 xmppConnectionService.sendIqPacket(
2833 id.account,
2834 request,
2835 (response) -> {
2836 final var iceServers = IceServers.parse(response);
2837 if (iceServers.isEmpty()) {
2838 Log.w(
2839 Config.LOGTAG,
2840 id.account.getJid().asBareJid()
2841 + ": no ICE server found "
2842 + response);
2843 }
2844 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2845 });
2846 } else {
2847 Log.w(
2848 Config.LOGTAG,
2849 id.account.getJid().asBareJid() + ": has no external service discovery");
2850 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2851 }
2852 }
2853
2854 @Override
2855 protected void terminateTransport() {
2856 this.webRTCWrapper.close();
2857 }
2858
2859 @Override
2860 protected void finish() {
2861 if (isTerminated()) {
2862 this.cancelRingingTimeout();
2863 this.callIntegration.verifyDisconnected();
2864 this.webRTCWrapper.verifyClosed();
2865 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2866 super.finish();
2867 try {
2868 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2869 log.getParentFile().mkdirs();
2870 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2871 } catch (final IOException e) { }
2872 } else {
2873 throw new IllegalStateException(
2874 String.format("Unable to call finish from %s", this.state));
2875 }
2876 }
2877
2878 private void writeLogMessage(final State state) {
2879 final long duration = getCallDuration();
2880 if (state == State.TERMINATED_SUCCESS
2881 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2882 writeLogMessageSuccess(duration);
2883 } else {
2884 writeLogMessageMissed();
2885 }
2886 }
2887
2888 private void writeLogMessageSuccess(final long duration) {
2889 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2890 this.writeMessage();
2891 }
2892
2893 private void writeLogMessageMissed() {
2894 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2895 this.writeMessage();
2896 }
2897
2898 private void writeMessage() {
2899 final Conversational conversational = message.getConversation();
2900 if (conversational instanceof Conversation) {
2901 ((Conversation) conversational).add(this.message);
2902 xmppConnectionService.createMessageAsync(message);
2903 xmppConnectionService.updateConversationUi();
2904 } else {
2905 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2906 }
2907 }
2908
2909 public Optional<VideoTrack> getLocalVideoTrack() {
2910 return webRTCWrapper.getLocalVideoTrack();
2911 }
2912
2913 public Optional<VideoTrack> getRemoteVideoTrack() {
2914 return webRTCWrapper.getRemoteVideoTrack();
2915 }
2916
2917 public EglBase.Context getEglBaseContext() {
2918 return webRTCWrapper.getEglBaseContext();
2919 }
2920
2921 void setProposedMedia(final Set<Media> media) {
2922 this.proposedMedia = media;
2923 this.callIntegration.setVideoState(
2924 Media.audioOnly(media)
2925 ? VideoProfile.STATE_AUDIO_ONLY
2926 : VideoProfile.STATE_BIDIRECTIONAL);
2927 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2928 }
2929
2930 public void fireStateUpdate() {
2931 final RtpEndUserState endUserState = getEndUserState();
2932 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2933 id.account, id.with, id.sessionId, endUserState);
2934 }
2935
2936 public boolean isSwitchToVideoAvailable() {
2937 final boolean prerequisite =
2938 Media.audioOnly(getMedia())
2939 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2940 .contains(getEndUserState());
2941 return prerequisite && remoteHasVideoFeature();
2942 }
2943
2944 private boolean remoteHasVideoFeature() {
2945 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2946 }
2947
2948 private boolean remoteHasSdpOfferAnswer() {
2949 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2950 }
2951
2952 @Override
2953 public Account getAccount() {
2954 return id.account;
2955 }
2956
2957 @Override
2958 public Jid getWith() {
2959 return id.with;
2960 }
2961
2962 @Override
2963 public String getSessionId() {
2964 return id.sessionId;
2965 }
2966
2967 private interface OnIceServersDiscovered {
2968 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2969 }
2970}