1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
51import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
52
53import im.conversations.android.xmpp.model.jingle.Jingle;
54import im.conversations.android.xmpp.model.stanza.Iq;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 final var contact = id.getContact();
122 this.callIntegration.setCallerDisplayName(
123 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
124 this.callIntegration.setInitialized();
125 }
126
127 JingleRtpConnection(
128 final JingleConnectionManager jingleConnectionManager,
129 final Id id,
130 final Jid initiator,
131 final CallIntegration callIntegration) {
132 super(jingleConnectionManager, id, initiator);
133 final Conversation conversation =
134 jingleConnectionManager
135 .getXmppConnectionService()
136 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
137 this.message =
138 new Message(
139 conversation,
140 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
141 Message.TYPE_RTP_SESSION,
142 id.sessionId);
143 this.callIntegration = callIntegration;
144 this.callIntegration.setCallback(this);
145 }
146
147 @Override
148 synchronized void deliverPacket(final Iq iq) {
149 final var jingle = iq.getExtension(Jingle.class);
150 switch (jingle.getAction()) {
151 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
152 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
153 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
154 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
155 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
156 case CONTENT_ACCEPT -> receiveContentAccept(iq);
157 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
158 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
159 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
160 default -> {
161 respondOk(iq);
162 Log.d(
163 Config.LOGTAG,
164 String.format(
165 "%s: received unhandled jingle action %s",
166 id.account.getJid().asBareJid(), jingle.getAction()));
167 }
168 }
169 }
170
171 @Override
172 synchronized void notifyRebound() {
173 if (isTerminated()) {
174 return;
175 }
176 webRTCWrapper.close();
177 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
178 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
179 }
180 if (isInState(
181 State.SESSION_INITIALIZED,
182 State.SESSION_INITIALIZED_PRE_APPROVED,
183 State.SESSION_ACCEPTED)) {
184 // we might have already changed resources (full jid) at this point; so this might not
185 // even reach the other party
186 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
187 } else {
188 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
189 finish();
190 }
191 }
192
193 public boolean applyDtmfTone(final String tone) {
194 return webRTCWrapper.applyDtmfTone(tone);
195 }
196
197 private void receiveSessionTerminate(final Iq jinglePacket) {
198 respondOk(jinglePacket);
199 final var jingle = jinglePacket.getExtension(Jingle.class);
200 final Jingle.ReasonWrapper wrapper = jingle.getReason();
201 final State previous = this.state;
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": received session terminate reason="
206 + wrapper.reason
207 + "("
208 + Strings.nullToEmpty(wrapper.text)
209 + ") while in state "
210 + previous);
211 if (TERMINATED.contains(previous)) {
212 Log.d(
213 Config.LOGTAG,
214 id.account.getJid().asBareJid()
215 + ": ignoring session terminate because already in "
216 + previous);
217 return;
218 }
219 webRTCWrapper.close();
220 final State target = reasonToState(wrapper.reason);
221 transitionOrThrow(target);
222 writeLogMessage(target);
223 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 finish();
227 }
228
229 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
230 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
231 // INITIALIZED only after transport-info has been received
232 if (isInState(
233 State.NULL,
234 State.PROCEED,
235 State.SESSION_INITIALIZED,
236 State.SESSION_INITIALIZED_PRE_APPROVED,
237 State.SESSION_ACCEPTED)) {
238 final RtpContentMap contentMap;
239 try {
240 contentMap = RtpContentMap.of(jingle);
241 } catch (final IllegalArgumentException | NullPointerException e) {
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": improperly formatted contents; ignoring",
246 e);
247 respondOk(jinglePacket);
248 return;
249 }
250 receiveTransportInfo(jinglePacket, contentMap);
251 } else {
252 if (isTerminated()) {
253 respondOk(jinglePacket);
254 Log.d(
255 Config.LOGTAG,
256 id.account.getJid().asBareJid()
257 + ": ignoring out-of-order transport info; we where already terminated");
258 } else {
259 Log.d(
260 Config.LOGTAG,
261 id.account.getJid().asBareJid()
262 + ": received transport info while in state="
263 + this.state);
264 terminateWithOutOfOrder(jinglePacket);
265 }
266 }
267 }
268
269 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
270 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
271 candidates = contentMap.contents.entrySet();
272 final RtpContentMap remote = getRemoteContentMap();
273 final Set<String> remoteContentIds =
274 remote == null ? Collections.emptySet() : remote.contents.keySet();
275 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
276 Log.d(
277 Config.LOGTAG,
278 "received transport-info for unknown contents "
279 + contentMap.contents.keySet()
280 + " (known: "
281 + remoteContentIds
282 + ")");
283 respondOk(jinglePacket);
284 pendingIceCandidates.addAll(candidates);
285 return;
286 }
287 if (this.state != State.SESSION_ACCEPTED) {
288 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
289 respondOk(jinglePacket);
290 pendingIceCandidates.addAll(candidates);
291 return;
292 }
293 // zero candidates + modified credentials are an ICE restart offer
294 if (checkForIceRestart(jinglePacket, contentMap)) {
295 return;
296 }
297 respondOk(jinglePacket);
298 try {
299 processCandidates(candidates);
300 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
301 Log.w(
302 Config.LOGTAG,
303 id.account.getJid().asBareJid()
304 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
305 }
306 }
307
308 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
309 final RtpContentMap modification;
310 try {
311 modification = RtpContentMap.of(jingle);
312 modification.requireContentDescriptions();
313 } catch (final RuntimeException e) {
314 Log.d(
315 Config.LOGTAG,
316 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
317 Throwables.getRootCause(e));
318 respondOk(iq);
319 webRTCWrapper.close();
320 sendSessionTerminate(Reason.of(e), e.getMessage());
321 return;
322 }
323 if (isInState(State.SESSION_ACCEPTED)) {
324 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
325 final ListenableFuture<RtpContentMap> future =
326 receiveRtpContentMap(
327 modification,
328 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
329 Futures.addCallback(
330 future,
331 new FutureCallback<>() {
332 @Override
333 public void onSuccess(final RtpContentMap rtpContentMap) {
334 receiveContentAdd(iq, rtpContentMap);
335 }
336
337 @Override
338 public void onFailure(@NonNull Throwable throwable) {
339 respondOk(iq);
340 final Throwable rootCause = Throwables.getRootCause(throwable);
341 Log.d(
342 Config.LOGTAG,
343 id.account.getJid().asBareJid()
344 + ": improperly formatted contents in content-add",
345 throwable);
346 webRTCWrapper.close();
347 sendSessionTerminate(
348 Reason.ofThrowable(rootCause), rootCause.getMessage());
349 }
350 },
351 MoreExecutors.directExecutor());
352 } else {
353 terminateWithOutOfOrder(iq);
354 }
355 }
356
357 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
358 final RtpContentMap remote = getRemoteContentMap();
359 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
360 respondOk(jinglePacket);
361 this.webRTCWrapper.close();
362 sendSessionTerminate(
363 Reason.FAILED_APPLICATION,
364 String.format(
365 "contents with names %s already exists",
366 Joiner.on(", ").join(modification.getNames())));
367 return;
368 }
369 final ContentAddition contentAddition =
370 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
371
372 final RtpContentMap outgoing = this.outgoingContentAdd;
373 final Set<ContentAddition.Summary> outgoingContentAddSummary =
374 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
375
376 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
377 if (isInitiator()) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid()
381 + ": respond with tie break to matching content-add offer");
382 respondWithTieBreak(jinglePacket);
383 } else {
384 Log.d(
385 Config.LOGTAG,
386 id.getAccount().getJid().asBareJid()
387 + ": automatically accept matching content-add offer");
388 acceptContentAdd(contentAddition.summary, modification);
389 }
390 return;
391 }
392
393 // once we can display multiple video tracks we can be more loose with this condition
394 // theoretically it should also be fine to automatically accept audio only contents
395 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
396 Log.d(
397 Config.LOGTAG,
398 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
399 this.incomingContentAdd = modification;
400 respondOk(jinglePacket);
401 updateEndUserState();
402 } else {
403 respondOk(jinglePacket);
404 // TODO do we want to add a reason?
405 rejectContentAdd(modification);
406 }
407 }
408
409 private void receiveContentAccept(final Iq jinglePacket) {
410 final var jingle = jinglePacket.getExtension(Jingle.class);
411 final RtpContentMap receivedContentAccept;
412 try {
413 receivedContentAccept = RtpContentMap.of(jingle);
414 receivedContentAccept.requireContentDescriptions();
415 } catch (final RuntimeException e) {
416 Log.d(
417 Config.LOGTAG,
418 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
419 Throwables.getRootCause(e));
420 respondOk(jinglePacket);
421 webRTCWrapper.close();
422 sendSessionTerminate(Reason.of(e), e.getMessage());
423 return;
424 }
425
426 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
427 if (outgoingContentAdd == null) {
428 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
429 terminateWithOutOfOrder(jinglePacket);
430 return;
431 }
432 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
433 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
434 this.outgoingContentAdd = null;
435 respondOk(jinglePacket);
436 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
437 final ListenableFuture<RtpContentMap> future =
438 receiveRtpContentMap(
439 receivedContentAccept,
440 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
441 Futures.addCallback(
442 future,
443 new FutureCallback<>() {
444 @Override
445 public void onSuccess(final RtpContentMap result) {
446 receiveContentAccept(result);
447 }
448
449 @Override
450 public void onFailure(@NonNull final Throwable throwable) {
451 webRTCWrapper.close();
452 sendSessionTerminate(
453 Reason.ofThrowable(throwable), throwable.getMessage());
454 }
455 },
456 MoreExecutors.directExecutor());
457 } else {
458 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
459 terminateWithOutOfOrder(jinglePacket);
460 }
461 }
462
463 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
464 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
465 final RtpContentMap modifiedContentMap =
466 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
467
468 setRemoteContentMap(modifiedContentMap);
469
470 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
471
472 final org.webrtc.SessionDescription sdp =
473 new org.webrtc.SessionDescription(
474 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
475
476 try {
477 this.webRTCWrapper.setRemoteDescription(sdp).get();
478 } catch (final Exception e) {
479 final Throwable cause = Throwables.getRootCause(e);
480 Log.d(
481 Config.LOGTAG,
482 id.getAccount().getJid().asBareJid()
483 + ": unable to set remote description after receiving content-accept",
484 cause);
485 webRTCWrapper.close();
486 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
487 return;
488 }
489 Log.d(
490 Config.LOGTAG,
491 id.getAccount().getJid().asBareJid()
492 + ": remote has accepted content-add "
493 + ContentAddition.summary(receivedContentAccept));
494 processCandidates(receivedContentAccept.contents.entrySet());
495 updateEndUserState();
496 }
497
498 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
499 if (this.state != State.SESSION_ACCEPTED) {
500 terminateWithOutOfOrder(jinglePacket);
501 return;
502 }
503 final Map<String, Content.Senders> modification =
504 Maps.transformEntries(
505 jingle.getJingleContents(), (key, value) -> value.getSenders());
506 final boolean isInitiator = isInitiator();
507 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
508 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
509 final Set<String> currentOutgoingMediaIds =
510 currentOutgoing == null
511 ? Collections.emptySet()
512 : currentOutgoing.contents.keySet();
513 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
514 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
515 respondOk(jinglePacket);
516 final RtpContentMap modifiedContentMap;
517 try {
518 modifiedContentMap =
519 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
520 } catch (final IllegalArgumentException e) {
521 webRTCWrapper.close();
522 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
523 return;
524 }
525 this.outgoingContentAdd = modifiedContentMap;
526 Log.d(
527 Config.LOGTAG,
528 id.account.getJid().asBareJid()
529 + ": processed content-modification for pending content-add");
530 } else if (remoteContentMap != null
531 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
532 respondOk(jinglePacket);
533 final RtpContentMap modifiedRemoteContentMap;
534 try {
535 modifiedRemoteContentMap =
536 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
537 } catch (final IllegalArgumentException e) {
538 webRTCWrapper.close();
539 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
540 return;
541 }
542 final SessionDescription offer;
543 try {
544 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
545 } catch (final IllegalArgumentException | NullPointerException e) {
546 Log.d(
547 Config.LOGTAG,
548 id.getAccount().getJid().asBareJid()
549 + ": unable convert offer from content-modify to SDP",
550 e);
551 webRTCWrapper.close();
552 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
553 return;
554 }
555 Log.d(
556 Config.LOGTAG,
557 id.account.getJid().asBareJid() + ": auto accepting content-modification");
558 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
559 } else {
560 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
561 respondWithItemNotFound(jinglePacket);
562 }
563 }
564
565 private void autoAcceptContentModify(
566 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
567 this.setRemoteContentMap(modifiedRemoteContentMap);
568 final org.webrtc.SessionDescription sdp =
569 new org.webrtc.SessionDescription(
570 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
571 try {
572 this.webRTCWrapper.setRemoteDescription(sdp).get();
573 // auto accept is only done when we already have tracks
574 final SessionDescription answer = setLocalSessionDescription();
575 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
576 modifyLocalContentMap(rtpContentMap);
577 // we do not need to send an answer but do we have to resend the candidates currently in
578 // SDP?
579 // resendCandidatesFromSdp(answer);
580 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
581 } catch (final Exception e) {
582 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
583 webRTCWrapper.close();
584 sendSessionTerminate(Reason.FAILED_APPLICATION);
585 }
586 }
587
588 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
589 final SessionDescription answer) {
590 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
591 new ImmutableMultimap.Builder<>();
592 for (final SessionDescription.Media media : answer.media) {
593 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
594 if (Strings.isNullOrEmpty(mid)) {
595 continue;
596 }
597 for (final String sdpCandidate : media.attributes.get("candidate")) {
598 final IceUdpTransportInfo.Candidate candidate =
599 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
600 if (candidate != null) {
601 candidateBuilder.put(mid, candidate);
602 }
603 }
604 }
605 return candidateBuilder.build();
606 }
607
608 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
609 final RtpContentMap receivedContentReject;
610 try {
611 receivedContentReject = RtpContentMap.of(jingle);
612 } catch (final RuntimeException e) {
613 Log.d(
614 Config.LOGTAG,
615 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
616 Throwables.getRootCause(e));
617 respondOk(jinglePacket);
618 this.webRTCWrapper.close();
619 sendSessionTerminate(Reason.of(e), e.getMessage());
620 return;
621 }
622
623 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
624 if (outgoingContentAdd == null) {
625 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
626 terminateWithOutOfOrder(jinglePacket);
627 return;
628 }
629 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
630 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
631 this.outgoingContentAdd = null;
632 respondOk(jinglePacket);
633 Log.d(Config.LOGTAG, jinglePacket.toString());
634 receiveContentReject(ourSummary);
635 } else {
636 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
637 terminateWithOutOfOrder(jinglePacket);
638 }
639 }
640
641 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
642 try {
643 this.webRTCWrapper.removeTrack(Media.VIDEO);
644 final RtpContentMap localContentMap = customRollback();
645 modifyLocalContentMap(localContentMap);
646 } catch (final Exception e) {
647 final Throwable cause = Throwables.getRootCause(e);
648 Log.d(
649 Config.LOGTAG,
650 id.getAccount().getJid().asBareJid()
651 + ": unable to rollback local description after receiving content-reject",
652 cause);
653 webRTCWrapper.close();
654 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
655 return;
656 }
657 Log.d(
658 Config.LOGTAG,
659 id.getAccount().getJid().asBareJid()
660 + ": remote has rejected our content-add "
661 + summary);
662 }
663
664 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
665 final RtpContentMap receivedContentRemove;
666 try {
667 receivedContentRemove = RtpContentMap.of(jingle);
668 receivedContentRemove.requireContentDescriptions();
669 } catch (final RuntimeException e) {
670 Log.d(
671 Config.LOGTAG,
672 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
673 Throwables.getRootCause(e));
674 respondOk(jinglePacket);
675 this.webRTCWrapper.close();
676 sendSessionTerminate(Reason.of(e), e.getMessage());
677 return;
678 }
679 respondOk(jinglePacket);
680 receiveContentRemove(receivedContentRemove);
681 }
682
683 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
684 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
685 final Set<ContentAddition.Summary> contentAddSummary =
686 incomingContentAdd == null
687 ? Collections.emptySet()
688 : ContentAddition.summary(incomingContentAdd);
689 final Set<ContentAddition.Summary> removeSummary =
690 ContentAddition.summary(receivedContentRemove);
691 if (contentAddSummary.equals(removeSummary)) {
692 this.incomingContentAdd = null;
693 updateEndUserState();
694 } else {
695 webRTCWrapper.close();
696 sendSessionTerminate(
697 Reason.FAILED_APPLICATION,
698 String.format(
699 "%s only supports %s as a means to retract a not yet accepted %s",
700 BuildConfig.APP_NAME,
701 Jingle.Action.CONTENT_REMOVE,
702 Jingle.Action.CONTENT_ADD));
703 }
704 }
705
706 public synchronized void retractContentAdd() {
707 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
708 if (outgoingContentAdd == null) {
709 throw new IllegalStateException("Not outgoing content add");
710 }
711 try {
712 webRTCWrapper.removeTrack(Media.VIDEO);
713 final RtpContentMap localContentMap = customRollback();
714 modifyLocalContentMap(localContentMap);
715 } catch (final Exception e) {
716 final Throwable cause = Throwables.getRootCause(e);
717 Log.d(
718 Config.LOGTAG,
719 id.getAccount().getJid().asBareJid()
720 + ": unable to rollback local description after trying to retract content-add",
721 cause);
722 webRTCWrapper.close();
723 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
724 return;
725 }
726 this.outgoingContentAdd = null;
727 final Iq retract =
728 outgoingContentAdd
729 .toStub()
730 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
731 this.send(retract);
732 Log.d(
733 Config.LOGTAG,
734 id.getAccount().getJid()
735 + ": retract content-add "
736 + ContentAddition.summary(outgoingContentAdd));
737 }
738
739 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
740 final SessionDescription sdp = setLocalSessionDescription();
741 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
742 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
743 this.webRTCWrapper
744 .setRemoteDescription(
745 new org.webrtc.SessionDescription(
746 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
747 .get();
748 return localRtpContentMap;
749 }
750
751 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
752 final RtpContentMap currentRemote = getRemoteContentMap();
753 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
754 if (diff.isEmpty()) {
755 throw new IllegalStateException(
756 "Unexpected rollback condition. No difference between local and remote");
757 }
758 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
759 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
760 final RtpContentMap nextRemote =
761 currentRemote.addContent(
762 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
763 return SessionDescription.of(nextRemote, isResponder());
764 }
765 throw new IllegalStateException(
766 "Unexpected rollback condition. Senders were not uniformly none");
767 }
768
769 public synchronized void acceptContentAdd(
770 @NonNull final Set<ContentAddition.Summary> contentAddition) {
771 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
772 if (incomingContentAdd == null) {
773 throw new IllegalStateException("No incoming content add");
774 }
775
776 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
777 this.incomingContentAdd = null;
778 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
779 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
780 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
781 Log.d(
782 Config.LOGTAG,
783 "content addition is receive only. we want to upgrade to 'both'");
784 final RtpContentMap modifiedSenders =
785 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
786 final Iq proposedContentModification =
787 modifiedSenders
788 .toStub()
789 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
790 proposedContentModification.setTo(id.with);
791 xmppConnectionService.sendIqPacket(
792 id.account,
793 proposedContentModification,
794 (response) -> {
795 if (response.getType() == Iq.Type.RESULT) {
796 Log.d(
797 Config.LOGTAG,
798 id.account.getJid().asBareJid()
799 + ": remote has accepted our upgrade to senders=both");
800 acceptContentAdd(
801 ContentAddition.summary(modifiedSenders), modifiedSenders);
802 } else {
803 Log.d(
804 Config.LOGTAG,
805 id.account.getJid().asBareJid()
806 + ": remote has rejected our upgrade to senders=both");
807 acceptContentAdd(contentAddition, incomingContentAdd);
808 }
809 });
810 } else {
811 acceptContentAdd(contentAddition, incomingContentAdd);
812 }
813 } else {
814 throw new IllegalStateException(
815 "Accepted content add does not match pending content-add");
816 }
817 }
818
819 private void acceptContentAdd(
820 @NonNull final Set<ContentAddition.Summary> contentAddition,
821 final RtpContentMap incomingContentAdd) {
822 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
823 final RtpContentMap modifiedContentMap =
824 getRemoteContentMap().addContent(incomingContentAdd, setup);
825 this.setRemoteContentMap(modifiedContentMap);
826
827 final SessionDescription offer;
828 try {
829 offer = SessionDescription.of(modifiedContentMap, isResponder());
830 } catch (final IllegalArgumentException | NullPointerException e) {
831 Log.d(
832 Config.LOGTAG,
833 id.getAccount().getJid().asBareJid()
834 + ": unable convert offer from content-add to SDP",
835 e);
836 webRTCWrapper.close();
837 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
838 return;
839 }
840 this.incomingContentAdd = null;
841 acceptContentAdd(contentAddition, offer);
842 }
843
844 private void acceptContentAdd(
845 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
846 final org.webrtc.SessionDescription sdp =
847 new org.webrtc.SessionDescription(
848 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
849 try {
850 this.webRTCWrapper.setRemoteDescription(sdp).get();
851
852 // TODO add tracks for 'media' where contentAddition.senders matches
853
854 // TODO if senders.sending(isInitiator())
855
856 this.webRTCWrapper.addTrack(Media.VIDEO);
857
858 // TODO add additional transceivers for recv only cases
859
860 final SessionDescription answer = setLocalSessionDescription();
861 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
862
863 final RtpContentMap contentAcceptMap =
864 rtpContentMap.toContentModification(
865 Collections2.transform(contentAddition, ca -> ca.name));
866
867 Log.d(
868 Config.LOGTAG,
869 id.getAccount().getJid().asBareJid()
870 + ": sending content-accept "
871 + ContentAddition.summary(contentAcceptMap));
872
873 addIceCandidatesFromBlackLog();
874
875 modifyLocalContentMap(rtpContentMap);
876 final ListenableFuture<RtpContentMap> future =
877 prepareOutgoingContentMap(contentAcceptMap);
878 Futures.addCallback(
879 future,
880 new FutureCallback<>() {
881 @Override
882 public void onSuccess(final RtpContentMap rtpContentMap) {
883 sendContentAccept(rtpContentMap);
884 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
885 }
886
887 @Override
888 public void onFailure(@NonNull final Throwable throwable) {
889 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
890 }
891 },
892 MoreExecutors.directExecutor());
893 } catch (final Exception e) {
894 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
895 webRTCWrapper.close();
896 sendSessionTerminate(Reason.FAILED_APPLICATION);
897 }
898 }
899
900 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
901 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
902 send(iq);
903 }
904
905 public synchronized void rejectContentAdd() {
906 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
907 if (incomingContentAdd == null) {
908 throw new IllegalStateException("No incoming content add");
909 }
910 this.incomingContentAdd = null;
911 updateEndUserState();
912 rejectContentAdd(incomingContentAdd);
913 }
914
915 private void rejectContentAdd(final RtpContentMap contentMap) {
916 final Iq iq =
917 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
918 Log.d(
919 Config.LOGTAG,
920 id.getAccount().getJid().asBareJid()
921 + ": rejecting content "
922 + ContentAddition.summary(contentMap));
923 send(iq);
924 }
925
926 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
927 final RtpContentMap existing = getRemoteContentMap();
928 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
929 final IceUdpTransportInfo.Credentials newCredentials;
930 try {
931 existingCredentials = existing.getCredentials();
932 newCredentials = rtpContentMap.getDistinctCredentials();
933 } catch (final IllegalStateException e) {
934 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
935 return false;
936 }
937 if (existingCredentials.contains(newCredentials)) {
938 return false;
939 }
940 // TODO an alternative approach is to check if we already got an iq result to our
941 // ICE-restart
942 // and if that's the case we are seeing an answer.
943 // This might be more spec compliant but also more error prone potentially
944 final boolean isSignalStateStable =
945 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
946 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
947 final boolean isOffer = rtpContentMap.emptyCandidates();
948 final RtpContentMap restartContentMap;
949 try {
950 if (isOffer) {
951 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
952 restartContentMap =
953 existing.modifiedCredentials(
954 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
955 } else {
956 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
957 Log.d(
958 Config.LOGTAG,
959 "received confirmation of ICE restart"
960 + newCredentials
961 + " peer_setup="
962 + setup);
963 // DTLS setup attribute needs to be rewritten to reflect current peer state
964 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
965 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
966 }
967 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
968 return isOffer;
969 } else {
970 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
971 respondWithTieBreak(jinglePacket);
972 return true;
973 }
974 } catch (final Exception exception) {
975 respondOk(jinglePacket);
976 final Throwable rootCause = Throwables.getRootCause(exception);
977 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
978 // If this happens a termination is already in progress
979 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
980 return true;
981 }
982 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
983 webRTCWrapper.close();
984 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
985 return true;
986 }
987 }
988
989 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
990 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
991 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
992 throw new IllegalStateException("Invalid peer setup");
993 }
994 return peerSetup;
995 }
996
997 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
998 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
999 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1000 }
1001 this.peerDtlsSetup = setup;
1002 }
1003
1004 private boolean applyIceRestart(
1005 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
1006 throws ExecutionException, InterruptedException {
1007 final SessionDescription sessionDescription =
1008 SessionDescription.of(restartContentMap, isResponder());
1009 final org.webrtc.SessionDescription.Type type =
1010 isOffer
1011 ? org.webrtc.SessionDescription.Type.OFFER
1012 : org.webrtc.SessionDescription.Type.ANSWER;
1013 org.webrtc.SessionDescription sdp =
1014 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1015 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1016 if (isInitiator()) {
1017 // We ignore the offer and respond with tie-break. This will clause the responder
1018 // not to apply the content map
1019 return false;
1020 }
1021 }
1022 webRTCWrapper.setRemoteDescription(sdp).get();
1023 setRemoteContentMap(restartContentMap);
1024 if (isOffer) {
1025 final SessionDescription localSessionDescription = setLocalSessionDescription();
1026 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1027 // We need to respond OK before sending any candidates
1028 respondOk(jinglePacket);
1029 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1030 } else {
1031 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1032 }
1033 return true;
1034 }
1035
1036 private void processCandidates(
1037 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1038 contents) {
1039 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1040 content : contents) {
1041 processCandidate(content);
1042 }
1043 }
1044
1045 private void processCandidate(
1046 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1047 content) {
1048 final RtpContentMap rtpContentMap = getRemoteContentMap();
1049 final List<String> indices = toIdentificationTags(rtpContentMap);
1050 final String sdpMid = content.getKey(); // aka content name
1051 final IceUdpTransportInfo transport = content.getValue().transport;
1052 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1053
1054 // TODO check that credentials remained the same
1055
1056 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1057 final String sdp;
1058 try {
1059 sdp = candidate.toSdpAttribute(credentials.ufrag);
1060 } catch (final IllegalArgumentException e) {
1061 Log.d(
1062 Config.LOGTAG,
1063 id.account.getJid().asBareJid()
1064 + ": ignoring invalid ICE candidate "
1065 + e.getMessage());
1066 continue;
1067 }
1068 final int mLineIndex = indices.indexOf(sdpMid);
1069 if (mLineIndex < 0) {
1070 Log.w(
1071 Config.LOGTAG,
1072 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1073 }
1074 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1075 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1076 this.webRTCWrapper.addIceCandidate(iceCandidate);
1077 }
1078 }
1079
1080 private RtpContentMap getRemoteContentMap() {
1081 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1082 }
1083
1084 private RtpContentMap getLocalContentMap() {
1085 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1086 }
1087
1088 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1089 final Group originalGroup = rtpContentMap.group;
1090 final List<String> identificationTags =
1091 originalGroup == null
1092 ? rtpContentMap.getNames()
1093 : originalGroup.getIdentificationTags();
1094 if (identificationTags.size() == 0) {
1095 Log.w(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1099 }
1100 return identificationTags;
1101 }
1102
1103 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1104 final Jingle jinglePacket, final boolean expectVerification) {
1105 try {
1106 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1107 } catch (final Exception e) {
1108 return Futures.immediateFailedFuture(e);
1109 }
1110 }
1111
1112 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1113 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1114 Log.d(
1115 Config.LOGTAG,
1116 "receiveRtpContentMap("
1117 + receivedContentMap.getClass().getSimpleName()
1118 + ",expectVerification="
1119 + expectVerification
1120 + ")");
1121 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1122 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1123 id.account
1124 .getAxolotlService()
1125 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1126 return Futures.transform(
1127 future,
1128 omemoVerifiedPayload -> {
1129 // TODO test if an exception here triggers a correct abort
1130 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1131 Log.d(
1132 Config.LOGTAG,
1133 id.account.getJid().asBareJid()
1134 + ": received verifiable DTLS fingerprint via "
1135 + omemoVerification);
1136 return omemoVerifiedPayload.getPayload();
1137 },
1138 MoreExecutors.directExecutor());
1139 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1140 return Futures.immediateFailedFuture(
1141 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1142 } else {
1143 return Futures.immediateFuture(receivedContentMap);
1144 }
1145 }
1146
1147 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1148 if (isInitiator()) {
1149 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1150 return;
1151 }
1152 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1153 Futures.addCallback(
1154 future,
1155 new FutureCallback<>() {
1156 @Override
1157 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1158 receiveSessionInitiate(jinglePacket, rtpContentMap);
1159 }
1160
1161 @Override
1162 public void onFailure(@NonNull final Throwable throwable) {
1163 respondOk(jinglePacket);
1164 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1165 }
1166 },
1167 MoreExecutors.directExecutor());
1168 }
1169
1170 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1171 try {
1172 contentMap.requireContentDescriptions();
1173 contentMap.requireDTLSFingerprint(true);
1174 } catch (final RuntimeException e) {
1175 Log.d(
1176 Config.LOGTAG,
1177 id.account.getJid().asBareJid() + ": improperly formatted contents",
1178 Throwables.getRootCause(e));
1179 respondOk(jinglePacket);
1180 sendSessionTerminate(Reason.of(e), e.getMessage());
1181 return;
1182 }
1183 Log.d(
1184 Config.LOGTAG,
1185 "processing session-init with " + contentMap.contents.size() + " contents");
1186 final State target;
1187 if (this.state == State.PROCEED) {
1188 Preconditions.checkState(
1189 proposedMedia != null && proposedMedia.size() > 0,
1190 "proposed media must be set when processing pre-approved session-initiate");
1191 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1192 sendSessionTerminate(
1193 Reason.SECURITY_ERROR,
1194 String.format(
1195 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1196 this.proposedMedia, contentMap.getMedia()));
1197 return;
1198 }
1199 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1200 } else {
1201 target = State.SESSION_INITIALIZED;
1202 setProposedMedia(contentMap.getMedia());
1203 }
1204 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1205 respondOk(jinglePacket);
1206 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1207 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1208 Log.d(
1209 Config.LOGTAG,
1210 id.account.getJid().asBareJid()
1211 + ": automatically accepting session-initiate");
1212 sendSessionAccept();
1213 } else {
1214 Log.d(
1215 Config.LOGTAG,
1216 id.account.getJid().asBareJid()
1217 + ": received not pre-approved session-initiate. start ringing");
1218 startRinging();
1219 }
1220 } else {
1221 Log.d(
1222 Config.LOGTAG,
1223 String.format(
1224 "%s: received session-initiate while in state %s",
1225 id.account.getJid().asBareJid(), state));
1226 terminateWithOutOfOrder(jinglePacket);
1227 }
1228 }
1229
1230 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1231 if (isResponder()) {
1232 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1233 return;
1234 }
1235 final ListenableFuture<RtpContentMap> future =
1236 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1237 Futures.addCallback(
1238 future,
1239 new FutureCallback<>() {
1240 @Override
1241 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1242 receiveSessionAccept(jinglePacket, rtpContentMap);
1243 }
1244
1245 @Override
1246 public void onFailure(@NonNull final Throwable throwable) {
1247 respondOk(jinglePacket);
1248 Log.d(
1249 Config.LOGTAG,
1250 id.account.getJid().asBareJid()
1251 + ": improperly formatted contents in session-accept",
1252 throwable);
1253 webRTCWrapper.close();
1254 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1255 }
1256 },
1257 MoreExecutors.directExecutor());
1258 }
1259
1260 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1261 try {
1262 contentMap.requireContentDescriptions();
1263 contentMap.requireDTLSFingerprint();
1264 } catch (final RuntimeException e) {
1265 respondOk(jinglePacket);
1266 Log.d(
1267 Config.LOGTAG,
1268 id.account.getJid().asBareJid()
1269 + ": improperly formatted contents in session-accept",
1270 e);
1271 webRTCWrapper.close();
1272 sendSessionTerminate(Reason.of(e), e.getMessage());
1273 return;
1274 }
1275 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1276 if (!initiatorMedia.equals(contentMap.getMedia())) {
1277 sendSessionTerminate(
1278 Reason.SECURITY_ERROR,
1279 String.format(
1280 "Your session-included included media %s but our session-initiate was %s",
1281 this.proposedMedia, contentMap.getMedia()));
1282 return;
1283 }
1284 Log.d(
1285 Config.LOGTAG,
1286 "processing session-accept with " + contentMap.contents.size() + " contents");
1287 if (transition(State.SESSION_ACCEPTED)) {
1288 respondOk(jinglePacket);
1289 receiveSessionAccept(contentMap);
1290 } else {
1291 Log.d(
1292 Config.LOGTAG,
1293 String.format(
1294 "%s: received session-accept while in state %s",
1295 id.account.getJid().asBareJid(), state));
1296 respondOk(jinglePacket);
1297 }
1298 }
1299
1300 private void receiveSessionAccept(final RtpContentMap contentMap) {
1301 this.responderRtpContentMap = contentMap;
1302 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1303 final SessionDescription sessionDescription;
1304 try {
1305 sessionDescription = SessionDescription.of(contentMap, false);
1306 } catch (final IllegalArgumentException | NullPointerException e) {
1307 Log.d(
1308 Config.LOGTAG,
1309 id.account.getJid().asBareJid()
1310 + ": unable convert offer from session-accept to SDP",
1311 e);
1312 webRTCWrapper.close();
1313 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1314 return;
1315 }
1316 final org.webrtc.SessionDescription answer =
1317 new org.webrtc.SessionDescription(
1318 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1319 try {
1320 this.webRTCWrapper.setRemoteDescription(answer).get();
1321 } catch (final Exception e) {
1322 Log.d(
1323 Config.LOGTAG,
1324 id.account.getJid().asBareJid()
1325 + ": unable to set remote description after receiving session-accept",
1326 Throwables.getRootCause(e));
1327 webRTCWrapper.close();
1328 sendSessionTerminate(
1329 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1330 return;
1331 }
1332 processCandidates(contentMap.contents.entrySet());
1333 }
1334
1335 private void sendSessionAccept() {
1336 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1337 if (rtpContentMap == null) {
1338 throw new IllegalStateException("initiator RTP Content Map has not been set");
1339 }
1340 final SessionDescription offer;
1341 try {
1342 offer = SessionDescription.of(rtpContentMap, true);
1343 } catch (final IllegalArgumentException | NullPointerException e) {
1344 Log.d(
1345 Config.LOGTAG,
1346 id.account.getJid().asBareJid()
1347 + ": unable convert offer from session-initiate to SDP",
1348 e);
1349 webRTCWrapper.close();
1350 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1351 return;
1352 }
1353 sendSessionAccept(rtpContentMap.getMedia(), offer);
1354 }
1355
1356 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1357 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1358 }
1359
1360 private synchronized void sendSessionAccept(
1361 final Set<Media> media,
1362 final SessionDescription offer,
1363 final List<PeerConnection.IceServer> iceServers) {
1364 if (isTerminated()) {
1365 Log.w(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid()
1368 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1369 return;
1370 }
1371 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1372 try {
1373 setupWebRTC(media, iceServers, !includeCandidates);
1374 } catch (final WebRTCWrapper.InitializationException e) {
1375 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1376 webRTCWrapper.close();
1377 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1378 return;
1379 }
1380 final org.webrtc.SessionDescription sdp =
1381 new org.webrtc.SessionDescription(
1382 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1383 try {
1384 this.webRTCWrapper.setRemoteDescription(sdp).get();
1385 addIceCandidatesFromBlackLog();
1386 org.webrtc.SessionDescription webRTCSessionDescription =
1387 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1388 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1389 } catch (final Exception e) {
1390 failureToAcceptSession(e);
1391 }
1392 }
1393
1394 private void failureToAcceptSession(final Throwable throwable) {
1395 if (isTerminated()) {
1396 return;
1397 }
1398 final Throwable rootCause = Throwables.getRootCause(throwable);
1399 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1400 webRTCWrapper.close();
1401 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1402 }
1403
1404 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1405 if (isTerminated()) {
1406 return;
1407 }
1408 final Throwable rootCause = Throwables.getRootCause(throwable);
1409 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1410 webRTCWrapper.close();
1411 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1412 }
1413
1414 private void addIceCandidatesFromBlackLog() {
1415 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1416 while ((foo = this.pendingIceCandidates.poll()) != null) {
1417 processCandidate(foo);
1418 Log.d(
1419 Config.LOGTAG,
1420 id.account.getJid().asBareJid() + ": added candidate from back log");
1421 }
1422 }
1423
1424 private void prepareSessionAccept(
1425 final org.webrtc.SessionDescription webRTCSessionDescription,
1426 final boolean includeCandidates) {
1427 final SessionDescription sessionDescription =
1428 SessionDescription.parse(webRTCSessionDescription.description);
1429 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1430 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1431 if (includeCandidates) {
1432 candidates = parseCandidates(sessionDescription);
1433 } else {
1434 candidates = ImmutableMultimap.of();
1435 }
1436 this.responderRtpContentMap = respondingRtpContentMap;
1437 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1438 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1439 prepareOutgoingContentMap(respondingRtpContentMap);
1440 Futures.addCallback(
1441 outgoingContentMapFuture,
1442 new FutureCallback<>() {
1443 @Override
1444 public void onSuccess(final RtpContentMap outgoingContentMap) {
1445 if (includeCandidates) {
1446 Log.d(
1447 Config.LOGTAG,
1448 "including "
1449 + candidates.size()
1450 + " candidates in session accept");
1451 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1452 } else {
1453 sendSessionAccept(outgoingContentMap);
1454 }
1455 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1456 }
1457
1458 @Override
1459 public void onFailure(@NonNull Throwable throwable) {
1460 failureToAcceptSession(throwable);
1461 }
1462 },
1463 MoreExecutors.directExecutor());
1464 }
1465
1466 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1467 if (isTerminated()) {
1468 Log.w(
1469 Config.LOGTAG,
1470 id.account.getJid().asBareJid()
1471 + ": preparing session accept was too slow. already terminated. nothing to do.");
1472 return;
1473 }
1474 transitionOrThrow(State.SESSION_ACCEPTED);
1475 final Iq sessionAccept =
1476 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1477 send(sessionAccept);
1478 }
1479
1480 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1481 final RtpContentMap rtpContentMap) {
1482 if (this.omemoVerification.hasDeviceId()) {
1483 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1484 verifiedPayloadFuture =
1485 id.account
1486 .getAxolotlService()
1487 .encrypt(
1488 rtpContentMap,
1489 id.with,
1490 omemoVerification.getDeviceId());
1491 return Futures.transform(
1492 verifiedPayloadFuture,
1493 verifiedPayload -> {
1494 omemoVerification.setOrEnsureEqual(verifiedPayload);
1495 return verifiedPayload.getPayload();
1496 },
1497 MoreExecutors.directExecutor());
1498 } else {
1499 return Futures.immediateFuture(rtpContentMap);
1500 }
1501 }
1502
1503 synchronized void deliveryMessage(
1504 final Jid from,
1505 final Element message,
1506 final String serverMessageId,
1507 final long timestamp) {
1508 Log.d(
1509 Config.LOGTAG,
1510 id.account.getJid().asBareJid()
1511 + ": delivered message to JingleRtpConnection "
1512 + message);
1513 switch (message.getName()) {
1514 case "propose" -> receivePropose(
1515 from, Propose.upgrade(message), serverMessageId, timestamp);
1516 case "proceed" -> receiveProceed(
1517 from, Proceed.upgrade(message), serverMessageId, timestamp);
1518 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1519 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1520 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1521 }
1522 }
1523
1524 void deliverFailedProceed(final String message) {
1525 Log.d(
1526 Config.LOGTAG,
1527 id.account.getJid().asBareJid()
1528 + ": receive message error for proceed message ("
1529 + Strings.nullToEmpty(message)
1530 + ")");
1531 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1532 webRTCWrapper.close();
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1536 this.finish();
1537 }
1538 }
1539
1540 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1541 final boolean originatedFromMyself =
1542 from.asBareJid().equals(id.account.getJid().asBareJid());
1543 if (originatedFromMyself) {
1544 if (transition(State.ACCEPTED)) {
1545 acceptedOnOtherDevice(serverMsgId, timestamp);
1546 } else {
1547 Log.d(
1548 Config.LOGTAG,
1549 id.account.getJid().asBareJid()
1550 + ": unable to transition to accept because already in state="
1551 + this.state);
1552 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1553 }
1554 } else {
1555 Log.d(
1556 Config.LOGTAG,
1557 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1558 }
1559 }
1560
1561 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1562 if (serverMsgId != null) {
1563 this.message.setServerMsgId(serverMsgId);
1564 }
1565 this.message.setTime(timestamp);
1566 this.message.setCarbon(true); // indicate that call was accepted on other device
1567 this.writeLogMessageSuccess(0);
1568 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1569 this.finish();
1570 }
1571
1572 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1573 final boolean originatedFromMyself =
1574 from.asBareJid().equals(id.account.getJid().asBareJid());
1575 // reject from another one of my clients
1576 if (originatedFromMyself) {
1577 receiveRejectFromMyself(serverMsgId, timestamp);
1578 } else if (isInitiator()) {
1579 if (from.equals(id.with)) {
1580 receiveRejectFromResponder();
1581 } else {
1582 Log.d(
1583 Config.LOGTAG,
1584 id.account.getJid()
1585 + ": ignoring reject from "
1586 + from
1587 + " for session with "
1588 + id.with);
1589 }
1590 } else {
1591 Log.d(
1592 Config.LOGTAG,
1593 id.account.getJid()
1594 + ": ignoring reject from "
1595 + from
1596 + " for session with "
1597 + id.with);
1598 }
1599 }
1600
1601 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1602 if (transition(State.REJECTED)) {
1603 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1604 this.finish();
1605 if (serverMsgId != null) {
1606 this.message.setServerMsgId(serverMsgId);
1607 }
1608 this.message.setTime(timestamp);
1609 this.message.setCarbon(true); // indicate that call was rejected on other device
1610 writeLogMessageMissed();
1611 } else {
1612 Log.d(
1613 Config.LOGTAG,
1614 "not able to transition into REJECTED because already in " + this.state);
1615 }
1616 }
1617
1618 private void receiveRejectFromResponder() {
1619 if (isInState(State.PROCEED)) {
1620 Log.d(
1621 Config.LOGTAG,
1622 id.account.getJid()
1623 + ": received reject while still in proceed. callee reconsidered");
1624 closeTransitionLogFinish(State.REJECTED_RACED);
1625 return;
1626 }
1627 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1628 Log.d(
1629 Config.LOGTAG,
1630 id.account.getJid()
1631 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1632 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1633 return;
1634 }
1635 Log.d(
1636 Config.LOGTAG,
1637 id.account.getJid()
1638 + ": ignoring reject from responder because already in state "
1639 + this.state);
1640 }
1641
1642 private void receivePropose(
1643 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1644 final boolean originatedFromMyself =
1645 from.asBareJid().equals(id.account.getJid().asBareJid());
1646 if (originatedFromMyself) {
1647 Log.d(
1648 Config.LOGTAG,
1649 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1650 } else if (transition(
1651 State.PROPOSED,
1652 () -> {
1653 final Collection<RtpDescription> descriptions =
1654 Collections2.transform(
1655 Collections2.filter(
1656 propose.getDescriptions(),
1657 d -> d instanceof RtpDescription),
1658 input -> (RtpDescription) input);
1659 final Collection<Media> media =
1660 Collections2.transform(descriptions, RtpDescription::getMedia);
1661 Preconditions.checkState(
1662 !media.contains(Media.UNKNOWN),
1663 "RTP descriptions contain unknown media");
1664 Log.d(
1665 Config.LOGTAG,
1666 id.account.getJid().asBareJid()
1667 + ": received session proposal from "
1668 + from
1669 + " for "
1670 + media);
1671 this.setProposedMedia(Sets.newHashSet(media));
1672 })) {
1673 if (serverMsgId != null) {
1674 this.message.setServerMsgId(serverMsgId);
1675 }
1676 this.message.setTime(timestamp);
1677 startRinging();
1678 // in environments where we always use discovery timeouts we always want to respond with
1679 // 'ringing'
1680 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1681 || (xmppConnectionService.confirmMessages()
1682 && id.getContact().showInContactList())) {
1683 sendJingleMessage("ringing");
1684 }
1685 } else {
1686 Log.d(
1687 Config.LOGTAG,
1688 id.account.getJid()
1689 + ": ignoring session proposal because already in "
1690 + state);
1691 }
1692 }
1693
1694 private void startRinging() {
1695 this.callIntegration.setRinging();
1696 Log.d(
1697 Config.LOGTAG,
1698 id.account.getJid().asBareJid()
1699 + ": received call from "
1700 + id.with
1701 + ". start ringing");
1702 ringingTimeoutFuture =
1703 jingleConnectionManager.schedule(
1704 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1705 if (CallIntegration.selfManaged(xmppConnectionService)) {
1706 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1707 return;
1708 }
1709 }
1710 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1711 }
1712
1713 private synchronized void ringingTimeout() {
1714 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1715 switch (this.state) {
1716 case PROPOSED -> {
1717 message.markUnread();
1718 rejectCallFromProposed();
1719 }
1720 case SESSION_INITIALIZED -> {
1721 message.markUnread();
1722 rejectCallFromSessionInitiate();
1723 }
1724 }
1725 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1726 }
1727
1728 private void cancelRingingTimeout() {
1729 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1730 if (future != null && !future.isCancelled()) {
1731 future.cancel(false);
1732 }
1733 }
1734
1735 private void receiveProceed(
1736 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1737 final Set<Media> media =
1738 Preconditions.checkNotNull(
1739 this.proposedMedia, "Proposed media has to be set before handling proceed");
1740 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1741 if (from.equals(id.with)) {
1742 if (isInitiator()) {
1743 if (transition(State.PROCEED)) {
1744 if (serverMsgId != null) {
1745 this.message.setServerMsgId(serverMsgId);
1746 }
1747 this.message.setTime(timestamp);
1748 final Integer remoteDeviceId = proceed.getDeviceId();
1749 if (isOmemoEnabled()) {
1750 this.omemoVerification.setDeviceId(remoteDeviceId);
1751 } else {
1752 if (remoteDeviceId != null) {
1753 Log.d(
1754 Config.LOGTAG,
1755 id.account.getJid().asBareJid()
1756 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1757 }
1758 this.omemoVerification.setDeviceId(null);
1759 }
1760 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1761 } else {
1762 Log.d(
1763 Config.LOGTAG,
1764 String.format(
1765 "%s: ignoring proceed because already in %s",
1766 id.account.getJid().asBareJid(), this.state));
1767 }
1768 } else {
1769 Log.d(
1770 Config.LOGTAG,
1771 String.format(
1772 "%s: ignoring proceed because we were not initializing",
1773 id.account.getJid().asBareJid()));
1774 }
1775 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1776 if (transition(State.ACCEPTED)) {
1777 Log.d(
1778 Config.LOGTAG,
1779 id.account.getJid().asBareJid()
1780 + ": moved session with "
1781 + id.with
1782 + " into state accepted after received carbon copied proceed");
1783 acceptedOnOtherDevice(serverMsgId, timestamp);
1784 }
1785 } else {
1786 Log.d(
1787 Config.LOGTAG,
1788 String.format(
1789 "%s: ignoring proceed from %s. was expected from %s",
1790 id.account.getJid().asBareJid(), from, id.with));
1791 }
1792 }
1793
1794 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1795 if (from.equals(id.with)) {
1796 final State target =
1797 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1798 if (transition(target)) {
1799 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1800 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1801 Log.d(
1802 Config.LOGTAG,
1803 id.account.getJid().asBareJid()
1804 + ": session with "
1805 + id.with
1806 + " has been retracted (serverMsgId="
1807 + serverMsgId
1808 + ")");
1809 if (serverMsgId != null) {
1810 this.message.setServerMsgId(serverMsgId);
1811 }
1812 this.message.setTime(timestamp);
1813 if (target == State.RETRACTED) {
1814 this.message.markUnread();
1815 }
1816 writeLogMessageMissed();
1817 finish();
1818 } else {
1819 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1820 }
1821 } else {
1822 // TODO parse retract from self
1823 Log.d(
1824 Config.LOGTAG,
1825 id.account.getJid().asBareJid()
1826 + ": received retract from "
1827 + from
1828 + ". expected retract from"
1829 + id.with
1830 + ". ignoring");
1831 }
1832 }
1833
1834 public void sendSessionInitiate() {
1835 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1836 }
1837
1838 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1839 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1840 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1841 }
1842
1843 private synchronized void sendSessionInitiate(
1844 final Set<Media> media,
1845 final State targetState,
1846 final List<PeerConnection.IceServer> iceServers) {
1847 if (isTerminated()) {
1848 Log.w(
1849 Config.LOGTAG,
1850 id.account.getJid().asBareJid()
1851 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1852 return;
1853 }
1854 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1855 try {
1856 setupWebRTC(media, iceServers, !includeCandidates);
1857 } catch (final WebRTCWrapper.InitializationException e) {
1858 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1859 webRTCWrapper.close();
1860 sendRetract(Reason.ofThrowable(e));
1861 return;
1862 }
1863 try {
1864 org.webrtc.SessionDescription webRTCSessionDescription =
1865 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1866 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1867 } catch (final Exception e) {
1868 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1869 // exceptions
1870 failureToInitiateSession(e, targetState);
1871 }
1872 }
1873
1874 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1875 if (isTerminated()) {
1876 return;
1877 }
1878 Log.d(
1879 Config.LOGTAG,
1880 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1881 Throwables.getRootCause(throwable));
1882 webRTCWrapper.close();
1883 final Reason reason = Reason.ofThrowable(throwable);
1884 if (isInState(targetState)) {
1885 sendSessionTerminate(reason, throwable.getMessage());
1886 } else {
1887 sendRetract(reason);
1888 }
1889 }
1890
1891 private void sendRetract(final Reason reason) {
1892 // TODO embed reason into retract
1893 sendJingleMessage("retract", id.with.asBareJid());
1894 transitionOrThrow(reasonToState(reason));
1895 this.finish();
1896 }
1897
1898 private void prepareSessionInitiate(
1899 final org.webrtc.SessionDescription webRTCSessionDescription,
1900 final boolean includeCandidates,
1901 final State targetState) {
1902 final SessionDescription sessionDescription =
1903 SessionDescription.parse(webRTCSessionDescription.description);
1904 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1905 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1906 if (includeCandidates) {
1907 candidates = parseCandidates(sessionDescription);
1908 } else {
1909 candidates = ImmutableMultimap.of();
1910 }
1911 this.initiatorRtpContentMap = rtpContentMap;
1912 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1913 encryptSessionInitiate(rtpContentMap);
1914 Futures.addCallback(
1915 outgoingContentMapFuture,
1916 new FutureCallback<>() {
1917 @Override
1918 public void onSuccess(final RtpContentMap outgoingContentMap) {
1919 if (includeCandidates) {
1920 Log.d(
1921 Config.LOGTAG,
1922 "including "
1923 + candidates.size()
1924 + " candidates in session initiate");
1925 sendSessionInitiate(
1926 outgoingContentMap.withCandidates(candidates), targetState);
1927 } else {
1928 sendSessionInitiate(outgoingContentMap, targetState);
1929 }
1930 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1931 }
1932
1933 @Override
1934 public void onFailure(@NonNull final Throwable throwable) {
1935 failureToInitiateSession(throwable, targetState);
1936 }
1937 },
1938 MoreExecutors.directExecutor());
1939 }
1940
1941 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1942 if (isTerminated()) {
1943 Log.w(
1944 Config.LOGTAG,
1945 id.account.getJid().asBareJid()
1946 + ": preparing session was too slow. already terminated. nothing to do.");
1947 return;
1948 }
1949 this.transitionOrThrow(targetState);
1950 final Iq sessionInitiate =
1951 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1952 send(sessionInitiate);
1953 }
1954
1955 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1956 final RtpContentMap rtpContentMap) {
1957 if (this.omemoVerification.hasDeviceId()) {
1958 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1959 verifiedPayloadFuture =
1960 id.account
1961 .getAxolotlService()
1962 .encrypt(
1963 rtpContentMap,
1964 id.with,
1965 omemoVerification.getDeviceId());
1966 final ListenableFuture<RtpContentMap> future =
1967 Futures.transform(
1968 verifiedPayloadFuture,
1969 verifiedPayload -> {
1970 omemoVerification.setSessionFingerprint(
1971 verifiedPayload.getFingerprint());
1972 return verifiedPayload.getPayload();
1973 },
1974 MoreExecutors.directExecutor());
1975 if (Config.REQUIRE_RTP_VERIFICATION) {
1976 return future;
1977 }
1978 return Futures.catching(
1979 future,
1980 CryptoFailedException.class,
1981 e -> {
1982 Log.w(
1983 Config.LOGTAG,
1984 id.account.getJid().asBareJid()
1985 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1986 e);
1987 return rtpContentMap;
1988 },
1989 MoreExecutors.directExecutor());
1990 } else {
1991 return Futures.immediateFuture(rtpContentMap);
1992 }
1993 }
1994
1995 protected void sendSessionTerminate(final Reason reason) {
1996 sendSessionTerminate(reason, null);
1997 }
1998
1999 protected void sendSessionTerminate(final Reason reason, final String text) {
2000 sendSessionTerminate(reason, text, this::writeLogMessage);
2001 sendJingleMessageFinish(reason);
2002 }
2003
2004 private void sendTransportInfo(
2005 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2006 final RtpContentMap transportInfo;
2007 try {
2008 final RtpContentMap rtpContentMap =
2009 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2010 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2011 } catch (final Exception e) {
2012 Log.d(
2013 Config.LOGTAG,
2014 id.account.getJid().asBareJid()
2015 + ": unable to prepare transport-info from candidate for content="
2016 + contentName);
2017 return;
2018 }
2019 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2020 send(iq);
2021 }
2022
2023 public RtpEndUserState getEndUserState() {
2024 switch (this.state) {
2025 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2026 if (isInitiator()) {
2027 return RtpEndUserState.RINGING;
2028 } else {
2029 return RtpEndUserState.INCOMING_CALL;
2030 }
2031 }
2032 case PROCEED -> {
2033 if (isInitiator()) {
2034 return RtpEndUserState.RINGING;
2035 } else {
2036 return RtpEndUserState.ACCEPTING_CALL;
2037 }
2038 }
2039 case SESSION_INITIALIZED_PRE_APPROVED -> {
2040 if (isInitiator()) {
2041 return RtpEndUserState.RINGING;
2042 } else {
2043 return RtpEndUserState.CONNECTING;
2044 }
2045 }
2046 case SESSION_ACCEPTED -> {
2047 final ContentAddition ca = getPendingContentAddition();
2048 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2049 return RtpEndUserState.INCOMING_CONTENT_ADD;
2050 }
2051 return getPeerConnectionStateAsEndUserState();
2052 }
2053 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2054 if (isInitiator()) {
2055 return RtpEndUserState.DECLINED_OR_BUSY;
2056 } else {
2057 return RtpEndUserState.ENDED;
2058 }
2059 }
2060 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2061 return RtpEndUserState.ENDED;
2062 }
2063 case RETRACTED_RACED -> {
2064 if (isInitiator()) {
2065 return RtpEndUserState.ENDED;
2066 } else {
2067 return RtpEndUserState.RETRACTED;
2068 }
2069 }
2070 case TERMINATED_CONNECTIVITY_ERROR -> {
2071 return zeroDuration()
2072 ? RtpEndUserState.CONNECTIVITY_ERROR
2073 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2074 }
2075 case TERMINATED_APPLICATION_FAILURE -> {
2076 return RtpEndUserState.APPLICATION_ERROR;
2077 }
2078 case TERMINATED_SECURITY_ERROR -> {
2079 return RtpEndUserState.SECURITY_ERROR;
2080 }
2081 }
2082 throw new IllegalStateException(
2083 String.format("%s has no equivalent EndUserState", this.state));
2084 }
2085
2086 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2087 final PeerConnection.PeerConnectionState state;
2088 try {
2089 state = webRTCWrapper.getState();
2090 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2091 // We usually close the WebRTCWrapper *before* transitioning so we might still
2092 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2093 return RtpEndUserState.ENDING_CALL;
2094 }
2095 return switch (state) {
2096 case CONNECTED -> RtpEndUserState.CONNECTED;
2097 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2098 case CLOSED -> RtpEndUserState.ENDING_CALL;
2099 default -> zeroDuration()
2100 ? RtpEndUserState.CONNECTIVITY_ERROR
2101 : RtpEndUserState.RECONNECTING;
2102 };
2103 }
2104
2105 private boolean isPeerConnectionConnected() {
2106 try {
2107 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2108 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2109 return false;
2110 }
2111 }
2112
2113 private void updateCallIntegrationState() {
2114 switch (this.state) {
2115 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2116 if (isInitiator()) {
2117 this.callIntegration.setDialing();
2118 } else {
2119 this.callIntegration.setRinging();
2120 }
2121 }
2122 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2123 if (isInitiator()) {
2124 this.callIntegration.setDialing();
2125 } else {
2126 this.callIntegration.setInitialized();
2127 }
2128 }
2129 case SESSION_ACCEPTED -> {
2130 if (isPeerConnectionConnected()) {
2131 this.callIntegration.setActive();
2132 } else {
2133 this.callIntegration.setInitialized();
2134 }
2135 }
2136 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2137 if (isInitiator()) {
2138 this.callIntegration.busy();
2139 } else {
2140 this.callIntegration.rejected();
2141 }
2142 }
2143 case TERMINATED_SUCCESS -> this.callIntegration.success();
2144 case ACCEPTED -> this.callIntegration.accepted();
2145 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2146 .retracted();
2147 case TERMINATED_CONNECTIVITY_ERROR,
2148 TERMINATED_APPLICATION_FAILURE,
2149 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2150 default -> throw new IllegalStateException(
2151 String.format("%s is not handled", this.state));
2152 }
2153 }
2154
2155 public ContentAddition getPendingContentAddition() {
2156 final RtpContentMap in = this.incomingContentAdd;
2157 final RtpContentMap out = this.outgoingContentAdd;
2158 if (out != null) {
2159 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2160 } else if (in != null) {
2161 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2162 } else {
2163 return null;
2164 }
2165 }
2166
2167 public Set<Media> getMedia() {
2168 final State current = getState();
2169 if (current == State.NULL) {
2170 if (isInitiator()) {
2171 return Preconditions.checkNotNull(
2172 this.proposedMedia, "RTP connection has not been initialized properly");
2173 }
2174 throw new IllegalStateException("RTP connection has not been initialized yet");
2175 }
2176 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2177 return Preconditions.checkNotNull(
2178 this.proposedMedia, "RTP connection has not been initialized properly");
2179 }
2180 final RtpContentMap localContentMap = getLocalContentMap();
2181 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2182 if (localContentMap != null) {
2183 return localContentMap.getMedia();
2184 } else if (initiatorContentMap != null) {
2185 return initiatorContentMap.getMedia();
2186 } else if (isTerminated()) {
2187 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2188 } else {
2189 return Preconditions.checkNotNull(
2190 this.proposedMedia, "RTP connection has not been initialized properly");
2191 }
2192 }
2193
2194 public boolean isVerified() {
2195 final String fingerprint = this.omemoVerification.getFingerprint();
2196 if (fingerprint == null) {
2197 return false;
2198 }
2199 final FingerprintStatus status =
2200 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2201 return status != null && status.isVerified();
2202 }
2203
2204 public boolean addMedia(final Media media) {
2205 final Set<Media> currentMedia = getMedia();
2206 if (currentMedia.contains(media)) {
2207 throw new IllegalStateException(String.format("%s has already been proposed", media));
2208 }
2209 // TODO add state protection - can only add while ACCEPTED or so
2210 Log.d(Config.LOGTAG, "adding media: " + media);
2211 return webRTCWrapper.addTrack(media);
2212 }
2213
2214 public synchronized void acceptCall() {
2215 switch (this.state) {
2216 case PROPOSED -> {
2217 cancelRingingTimeout();
2218 acceptCallFromProposed();
2219 }
2220 case SESSION_INITIALIZED -> {
2221 cancelRingingTimeout();
2222 acceptCallFromSessionInitialized();
2223 }
2224 case ACCEPTED -> Log.w(
2225 Config.LOGTAG,
2226 id.account.getJid().asBareJid()
2227 + ": the call has already been accepted with another client. UI was just lagging behind");
2228 case PROCEED, SESSION_ACCEPTED -> Log.w(
2229 Config.LOGTAG,
2230 id.account.getJid().asBareJid()
2231 + ": the call has already been accepted. user probably double tapped the UI");
2232 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2233 }
2234 }
2235
2236 public synchronized void rejectCall() {
2237 if (isTerminated()) {
2238 Log.w(
2239 Config.LOGTAG,
2240 id.account.getJid().asBareJid()
2241 + ": received rejectCall() when session has already been terminated. nothing to do");
2242 return;
2243 }
2244 switch (this.state) {
2245 case PROPOSED -> rejectCallFromProposed();
2246 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2247 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2248 }
2249 }
2250
2251 public synchronized void integrationFailure() {
2252 final var state = getState();
2253 if (state == State.PROPOSED) {
2254 Log.e(
2255 Config.LOGTAG,
2256 id.account.getJid().asBareJid()
2257 + ": failed call integration in state proposed");
2258 rejectCallFromProposed();
2259 } else if (state == State.SESSION_INITIALIZED) {
2260 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2261 this.webRTCWrapper.close();
2262 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2263 } else {
2264 throw new IllegalStateException(
2265 String.format("Can not fail integration in state %s", state));
2266 }
2267 }
2268
2269 public synchronized void endCall() {
2270 if (isTerminated()) {
2271 Log.w(
2272 Config.LOGTAG,
2273 id.account.getJid().asBareJid()
2274 + ": received endCall() when session has already been terminated. nothing to do");
2275 return;
2276 }
2277 if (isInState(State.PROPOSED) && isResponder()) {
2278 rejectCallFromProposed();
2279 return;
2280 }
2281 if (isInState(State.PROCEED)) {
2282 if (isInitiator()) {
2283 retractFromProceed();
2284 } else {
2285 rejectCallFromProceed();
2286 }
2287 return;
2288 }
2289 if (isInitiator()
2290 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2291 this.webRTCWrapper.close();
2292 sendSessionTerminate(Reason.CANCEL);
2293 return;
2294 }
2295 if (isInState(State.SESSION_INITIALIZED)) {
2296 rejectCallFromSessionInitiate();
2297 return;
2298 }
2299 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2300 this.webRTCWrapper.close();
2301 sendSessionTerminate(Reason.SUCCESS);
2302 return;
2303 }
2304 if (isInState(
2305 State.TERMINATED_APPLICATION_FAILURE,
2306 State.TERMINATED_CONNECTIVITY_ERROR,
2307 State.TERMINATED_DECLINED_OR_BUSY)) {
2308 Log.d(
2309 Config.LOGTAG,
2310 "ignoring request to end call because already in state " + this.state);
2311 return;
2312 }
2313 throw new IllegalStateException(
2314 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2315 }
2316
2317 private void retractFromProceed() {
2318 Log.d(Config.LOGTAG, "retract from proceed");
2319 this.sendJingleMessage("retract");
2320 closeTransitionLogFinish(State.RETRACTED_RACED);
2321 }
2322
2323 private void closeTransitionLogFinish(final State state) {
2324 this.webRTCWrapper.close();
2325 transitionOrThrow(state);
2326 writeLogMessage(state);
2327 finish();
2328 }
2329
2330 private void setupWebRTC(
2331 final Set<Media> media,
2332 final List<PeerConnection.IceServer> iceServers,
2333 final boolean trickle)
2334 throws WebRTCWrapper.InitializationException {
2335 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2336 this.webRTCWrapper.setup(this.xmppConnectionService);
2337 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2338 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2339 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2340 }
2341
2342 private void acceptCallFromProposed() {
2343 transitionOrThrow(State.PROCEED);
2344 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2345 this.callIntegration.startAudioRouting();
2346 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2347 this.sendJingleMessage("proceed");
2348 }
2349
2350 private void rejectCallFromProposed() {
2351 transitionOrThrow(State.REJECTED);
2352 writeLogMessageMissed();
2353 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2354 this.sendJingleMessage("reject");
2355 finish();
2356 }
2357
2358 private void rejectCallFromProceed() {
2359 this.sendJingleMessage("reject");
2360 closeTransitionLogFinish(State.REJECTED_RACED);
2361 }
2362
2363 private void rejectCallFromSessionInitiate() {
2364 webRTCWrapper.close();
2365 sendSessionTerminate(Reason.DECLINE);
2366 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2367 }
2368
2369 private void sendJingleMessage(final String action) {
2370 sendJingleMessage(action, id.with);
2371 }
2372
2373 private void sendJingleMessage(final String action, final Jid to) {
2374 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2375 messagePacket.setType(
2376 im.conversations.android.xmpp.model.stanza.Message.Type
2377 .CHAT); // we want to carbon copy those
2378 messagePacket.setTo(to);
2379 final Element intent =
2380 messagePacket
2381 .addChild(action, Namespace.JINGLE_MESSAGE)
2382 .setAttribute("id", id.sessionId);
2383 if ("proceed".equals(action)) {
2384 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2385 if (isOmemoEnabled()) {
2386 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2387 final Element device =
2388 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2389 device.setAttribute("id", deviceId);
2390 }
2391 }
2392 messagePacket.addChild("store", "urn:xmpp:hints");
2393 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2394 }
2395
2396 private void sendJingleMessageFinish(final Reason reason) {
2397 final var account = id.getAccount();
2398 final var messagePacket =
2399 xmppConnectionService
2400 .getMessageGenerator()
2401 .sessionFinish(id.with, id.sessionId, reason);
2402 xmppConnectionService.sendMessagePacket(account, messagePacket);
2403 }
2404
2405 private boolean isOmemoEnabled() {
2406 final Conversational conversational = message.getConversation();
2407 if (conversational instanceof Conversation) {
2408 return ((Conversation) conversational).getNextEncryption()
2409 == Message.ENCRYPTION_AXOLOTL;
2410 }
2411 return false;
2412 }
2413
2414 private void acceptCallFromSessionInitialized() {
2415 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2416 this.callIntegration.startAudioRouting();
2417 sendSessionAccept();
2418 }
2419
2420 @Override
2421 protected synchronized boolean transition(final State target, final Runnable runnable) {
2422 if (super.transition(target, runnable)) {
2423 updateEndUserState();
2424 updateOngoingCallNotification();
2425 return true;
2426 } else {
2427 return false;
2428 }
2429 }
2430
2431 @Override
2432 public void onIceCandidate(final IceCandidate iceCandidate) {
2433 final RtpContentMap rtpContentMap =
2434 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2435 final IceUdpTransportInfo.Credentials credentials;
2436 try {
2437 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2438 } catch (final IllegalArgumentException e) {
2439 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2440 return;
2441 }
2442 final String uFrag = credentials.ufrag;
2443 final IceUdpTransportInfo.Candidate candidate =
2444 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2445 if (candidate == null) {
2446 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2447 return;
2448 }
2449 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2450 sendTransportInfo(iceCandidate.sdpMid, candidate);
2451 }
2452
2453 @Override
2454 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2455 Log.d(
2456 Config.LOGTAG,
2457 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2458 this.stateHistory.add(newState);
2459 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2460 this.sessionDuration.start();
2461 updateOngoingCallNotification();
2462 } else if (this.sessionDuration.isRunning()) {
2463 this.sessionDuration.stop();
2464 updateOngoingCallNotification();
2465 }
2466
2467 final boolean neverConnected =
2468 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2469
2470 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2471 if (neverConnected) {
2472 if (isTerminated()) {
2473 Log.d(
2474 Config.LOGTAG,
2475 id.account.getJid().asBareJid()
2476 + ": not sending session-terminate after connectivity error because session is already in state "
2477 + this.state);
2478 return;
2479 }
2480 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2481 return;
2482 } else {
2483 this.restartIce();
2484 }
2485 }
2486 updateEndUserState();
2487 }
2488
2489 private void restartIce() {
2490 this.stateHistory.clear();
2491 this.webRTCWrapper.restartIceAsync();
2492 }
2493
2494 @Override
2495 public void onRenegotiationNeeded() {
2496 this.webRTCWrapper.execute(this::renegotiate);
2497 }
2498
2499 private synchronized void renegotiate() {
2500 final SessionDescription sessionDescription;
2501 try {
2502 sessionDescription = setLocalSessionDescription();
2503 } catch (final Exception e) {
2504 final Throwable cause = Throwables.getRootCause(e);
2505 webRTCWrapper.close();
2506 if (isTerminated()) {
2507 Log.d(
2508 Config.LOGTAG,
2509 "failed to renegotiate. session was already terminated",
2510 cause);
2511 return;
2512 }
2513 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2514 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2515 return;
2516 }
2517 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2518 final RtpContentMap currentContentMap = getLocalContentMap();
2519 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2520 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2521
2522 Log.d(
2523 Config.LOGTAG,
2524 id.getAccount().getJid().asBareJid()
2525 + ": renegotiate. iceRestart="
2526 + iceRestart
2527 + " content id diff="
2528 + diff);
2529
2530 if (diff.hasModifications() && iceRestart) {
2531 webRTCWrapper.close();
2532 sendSessionTerminate(
2533 Reason.FAILED_APPLICATION,
2534 "WebRTC unexpectedly tried to modify content and transport at once");
2535 return;
2536 }
2537
2538 if (iceRestart) {
2539 initiateIceRestart(rtpContentMap);
2540 return;
2541 } else if (diff.isEmpty()) {
2542 Log.d(
2543 Config.LOGTAG,
2544 "renegotiation. nothing to do. SignalingState="
2545 + this.webRTCWrapper.getSignalingState());
2546 }
2547
2548 if (diff.added.isEmpty()) {
2549 return;
2550 }
2551 modifyLocalContentMap(rtpContentMap);
2552 sendContentAdd(rtpContentMap, diff.added);
2553 }
2554
2555 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2556 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2557 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2558 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2559 iq.setTo(id.with);
2560 xmppConnectionService.sendIqPacket(
2561 id.account,
2562 iq,
2563 (response) -> {
2564 if (response.getType() == Iq.Type.RESULT) {
2565 Log.d(Config.LOGTAG, "received success to our ice restart");
2566 setLocalContentMap(rtpContentMap);
2567 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2568 return;
2569 }
2570 if (response.getType() == Iq.Type.ERROR) {
2571 if (isTieBreak(response)) {
2572 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2573 return;
2574 }
2575 handleIqErrorResponse(response);
2576 }
2577 if (response.getType() == Iq.Type.TIMEOUT) {
2578 handleIqTimeoutResponse(response);
2579 }
2580 });
2581 }
2582
2583 private boolean isTieBreak(final Iq response) {
2584 final Element error = response.findChild("error");
2585 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2586 }
2587
2588 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2589 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2590 this.outgoingContentAdd = contentAdd;
2591 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2592 prepareOutgoingContentMap(contentAdd);
2593 Futures.addCallback(
2594 outgoingContentMapFuture,
2595 new FutureCallback<>() {
2596 @Override
2597 public void onSuccess(final RtpContentMap outgoingContentMap) {
2598 sendContentAdd(outgoingContentMap);
2599 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2600 }
2601
2602 @Override
2603 public void onFailure(@NonNull Throwable throwable) {
2604 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2605 }
2606 },
2607 MoreExecutors.directExecutor());
2608 }
2609
2610 private void sendContentAdd(final RtpContentMap contentAdd) {
2611
2612 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2613 iq.setTo(id.with);
2614 xmppConnectionService.sendIqPacket(
2615 id.account,
2616 iq,
2617 (response) -> {
2618 if (response.getType() == Iq.Type.RESULT) {
2619 Log.d(
2620 Config.LOGTAG,
2621 id.getAccount().getJid().asBareJid()
2622 + ": received ACK to our content-add");
2623 return;
2624 }
2625 if (response.getType() == Iq.Type.ERROR) {
2626 if (isTieBreak(response)) {
2627 this.outgoingContentAdd = null;
2628 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2629 return;
2630 }
2631 handleIqErrorResponse(response);
2632 }
2633 if (response.getType() == Iq.Type.TIMEOUT) {
2634 handleIqTimeoutResponse(response);
2635 }
2636 });
2637 }
2638
2639 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2640 if (isInitiator()) {
2641 this.initiatorRtpContentMap = rtpContentMap;
2642 } else {
2643 this.responderRtpContentMap = rtpContentMap;
2644 }
2645 }
2646
2647 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2648 if (isInitiator()) {
2649 this.responderRtpContentMap = rtpContentMap;
2650 } else {
2651 this.initiatorRtpContentMap = rtpContentMap;
2652 }
2653 }
2654
2655 // this method is to be used for content map modifications that modify media
2656 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2657 final RtpContentMap activeContents = rtpContentMap.activeContents();
2658 setLocalContentMap(activeContents);
2659 this.callIntegration.setAudioDeviceWhenAvailable(
2660 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2661 updateEndUserState();
2662 }
2663
2664 private SessionDescription setLocalSessionDescription()
2665 throws ExecutionException, InterruptedException {
2666 final org.webrtc.SessionDescription sessionDescription =
2667 this.webRTCWrapper.setLocalDescription(false).get();
2668 return SessionDescription.parse(sessionDescription.description);
2669 }
2670
2671 private void closeWebRTCSessionAfterFailedConnection() {
2672 this.webRTCWrapper.close();
2673 synchronized (this) {
2674 if (isTerminated()) {
2675 Log.d(
2676 Config.LOGTAG,
2677 id.account.getJid().asBareJid()
2678 + ": no need to send session-terminate after failed connection. Other party already did");
2679 return;
2680 }
2681 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2682 }
2683 }
2684
2685 public boolean zeroDuration() {
2686 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2687 }
2688
2689 public long getCallDuration() {
2690 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2691 }
2692
2693 @Override
2694 public CallIntegration getCallIntegration() {
2695 return this.callIntegration;
2696 }
2697
2698 public boolean isMicrophoneEnabled() {
2699 return webRTCWrapper.isMicrophoneEnabled();
2700 }
2701
2702 public boolean setMicrophoneEnabled(final boolean enabled) {
2703 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2704 }
2705
2706 public boolean isVideoEnabled() {
2707 return webRTCWrapper.isVideoEnabled();
2708 }
2709
2710 public void setVideoEnabled(final boolean enabled) {
2711 webRTCWrapper.setVideoEnabled(enabled);
2712 }
2713
2714 public boolean isCameraSwitchable() {
2715 return webRTCWrapper.isCameraSwitchable();
2716 }
2717
2718 public boolean isFrontCamera() {
2719 return webRTCWrapper.isFrontCamera();
2720 }
2721
2722 public ListenableFuture<Boolean> switchCamera() {
2723 return webRTCWrapper.switchCamera();
2724 }
2725
2726 @Override
2727 public synchronized void onCallIntegrationShowIncomingCallUi() {
2728 if (isTerminated()) {
2729 // there might be race conditions with the call integration service invoking this
2730 // callback when the rtp session has already ended.
2731 Log.w(
2732 Config.LOGTAG,
2733 "CallIntegration requested incoming call UI but session was already terminated");
2734 return;
2735 }
2736 // TODO apparently this can be called too early as well?
2737 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2738 }
2739
2740 @Override
2741 public void onCallIntegrationDisconnect() {
2742 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2743 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2744 rejectCall();
2745 } else {
2746 endCall();
2747 }
2748 }
2749
2750 @Override
2751 public void onCallIntegrationReject() {
2752 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2753 try {
2754 rejectCall();
2755 } catch (final IllegalStateException e) {
2756 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2757 }
2758 }
2759
2760 @Override
2761 public void onCallIntegrationAnswer() {
2762 // we need to start the UI to a) show it and b) be able to ask for permissions
2763 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2764 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2765 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2766 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2767 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2768 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2769 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2770 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2771 xmppConnectionService.startActivity(intent);
2772 }
2773
2774 @Override
2775 public void onCallIntegrationSilence() {
2776 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2777 }
2778
2779 @Override
2780 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2781 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2782 // that was previous disabled by the user. A proper implementation would probably be to
2783 // track user choice and enable the microphone with a userEnabled() &&
2784 // callIntegration.isMicrophoneEnabled() condition
2785 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2786 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2787 }
2788
2789 @Override
2790 public void onAudioDeviceChanged(
2791 final CallIntegration.AudioDevice selectedAudioDevice,
2792 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2793 Log.d(
2794 Config.LOGTAG,
2795 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2796 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2797 selectedAudioDevice, availableAudioDevices);
2798 }
2799
2800 private void updateEndUserState() {
2801 final RtpEndUserState endUserState = getEndUserState();
2802 this.updateCallIntegrationState();
2803 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2804 id.account, id.with, id.sessionId, endUserState);
2805 }
2806
2807 private void updateOngoingCallNotification() {
2808 final State state = this.state;
2809 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2810 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2811 && isResponder()) {
2812 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2813 xmppConnectionService.removeOngoingCall();
2814 return;
2815 }
2816 final boolean reconnecting;
2817 if (state == State.SESSION_ACCEPTED) {
2818 reconnecting =
2819 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2820 } else {
2821 reconnecting = false;
2822 }
2823 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2824 } else {
2825 xmppConnectionService.removeOngoingCall();
2826 }
2827 }
2828
2829 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2830 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2831 final Iq request = new Iq(Iq.Type.GET);
2832 request.setTo(id.account.getDomain());
2833 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2834 xmppConnectionService.sendIqPacket(
2835 id.account,
2836 request,
2837 (response) -> {
2838 final var iceServers = IceServers.parse(response);
2839 if (iceServers.isEmpty()) {
2840 Log.w(
2841 Config.LOGTAG,
2842 id.account.getJid().asBareJid()
2843 + ": no ICE server found "
2844 + response);
2845 }
2846 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2847 });
2848 } else {
2849 Log.w(
2850 Config.LOGTAG,
2851 id.account.getJid().asBareJid() + ": has no external service discovery");
2852 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2853 }
2854 }
2855
2856 @Override
2857 protected void terminateTransport() {
2858 this.webRTCWrapper.close();
2859 }
2860
2861 @Override
2862 protected void finish() {
2863 if (isTerminated()) {
2864 this.cancelRingingTimeout();
2865 this.callIntegration.verifyDisconnected();
2866 this.webRTCWrapper.verifyClosed();
2867 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2868 super.finish();
2869 try {
2870 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2871 log.getParentFile().mkdirs();
2872 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2873 } catch (final IOException e) { }
2874 } else {
2875 throw new IllegalStateException(
2876 String.format("Unable to call finish from %s", this.state));
2877 }
2878 }
2879
2880 private void writeLogMessage(final State state) {
2881 final long duration = getCallDuration();
2882 if (state == State.TERMINATED_SUCCESS
2883 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2884 writeLogMessageSuccess(duration);
2885 } else {
2886 writeLogMessageMissed();
2887 }
2888 }
2889
2890 private void writeLogMessageSuccess(final long duration) {
2891 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2892 this.writeMessage();
2893 }
2894
2895 private void writeLogMessageMissed() {
2896 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2897 this.writeMessage();
2898 }
2899
2900 private void writeMessage() {
2901 final Conversational conversational = message.getConversation();
2902 if (conversational instanceof Conversation) {
2903 ((Conversation) conversational).add(this.message);
2904 xmppConnectionService.createMessageAsync(message);
2905 xmppConnectionService.updateConversationUi();
2906 } else {
2907 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2908 }
2909 }
2910
2911 public Optional<VideoTrack> getLocalVideoTrack() {
2912 return webRTCWrapper.getLocalVideoTrack();
2913 }
2914
2915 public Optional<VideoTrack> getRemoteVideoTrack() {
2916 return webRTCWrapper.getRemoteVideoTrack();
2917 }
2918
2919 public EglBase.Context getEglBaseContext() {
2920 return webRTCWrapper.getEglBaseContext();
2921 }
2922
2923 void setProposedMedia(final Set<Media> media) {
2924 this.proposedMedia = media;
2925 this.callIntegration.setVideoState(
2926 Media.audioOnly(media)
2927 ? VideoProfile.STATE_AUDIO_ONLY
2928 : VideoProfile.STATE_BIDIRECTIONAL);
2929 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2930 }
2931
2932 public void fireStateUpdate() {
2933 final RtpEndUserState endUserState = getEndUserState();
2934 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2935 id.account, id.with, id.sessionId, endUserState);
2936 }
2937
2938 public boolean isSwitchToVideoAvailable() {
2939 final boolean prerequisite =
2940 Media.audioOnly(getMedia())
2941 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2942 .contains(getEndUserState());
2943 return prerequisite && remoteHasVideoFeature();
2944 }
2945
2946 private boolean remoteHasVideoFeature() {
2947 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2948 }
2949
2950 private boolean remoteHasSdpOfferAnswer() {
2951 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2952 }
2953
2954 @Override
2955 public Account getAccount() {
2956 return id.account;
2957 }
2958
2959 @Override
2960 public Jid getWith() {
2961 return id.with;
2962 }
2963
2964 @Override
2965 public String getSessionId() {
2966 return id.sessionId;
2967 }
2968
2969 private interface OnIceServersDiscovered {
2970 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2971 }
2972}