1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
51import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
52
53import im.conversations.android.xmpp.model.jingle.Jingle;
54import im.conversations.android.xmpp.model.stanza.Iq;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 final var contact = id.getContact();
122 this.callIntegration.setCallerDisplayName(
123 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
124 this.callIntegration.setInitialized();
125 }
126
127 JingleRtpConnection(
128 final JingleConnectionManager jingleConnectionManager,
129 final Id id,
130 final Jid initiator,
131 final CallIntegration callIntegration) {
132 super(jingleConnectionManager, id, initiator);
133 final Conversation conversation =
134 jingleConnectionManager
135 .getXmppConnectionService()
136 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
137 this.message =
138 new Message(
139 conversation,
140 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
141 Message.TYPE_RTP_SESSION,
142 id.sessionId);
143 this.callIntegration = callIntegration;
144 this.callIntegration.setCallback(this);
145 }
146
147 @Override
148 synchronized void deliverPacket(final Iq iq) {
149 final var jingle = iq.getExtension(Jingle.class);
150 switch (jingle.getAction()) {
151 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
152 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
153 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
154 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
155 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
156 case CONTENT_ACCEPT -> receiveContentAccept(iq);
157 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
158 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
159 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
160 default -> {
161 respondOk(iq);
162 Log.d(
163 Config.LOGTAG,
164 String.format(
165 "%s: received unhandled jingle action %s",
166 id.account.getJid().asBareJid(), jingle.getAction()));
167 }
168 }
169 }
170
171 @Override
172 synchronized void notifyRebound() {
173 if (isTerminated()) {
174 return;
175 }
176 webRTCWrapper.close();
177 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
178 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
179 }
180 if (isInState(
181 State.SESSION_INITIALIZED,
182 State.SESSION_INITIALIZED_PRE_APPROVED,
183 State.SESSION_ACCEPTED)) {
184 // we might have already changed resources (full jid) at this point; so this might not
185 // even reach the other party
186 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
187 } else {
188 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
189 finish();
190 }
191 }
192
193 public boolean applyDtmfTone(final String tone) {
194 return webRTCWrapper.applyDtmfTone(tone);
195 }
196
197 private void receiveSessionTerminate(final Iq jinglePacket) {
198 respondOk(jinglePacket);
199 final var jingle = jinglePacket.getExtension(Jingle.class);
200 final Jingle.ReasonWrapper wrapper = jingle.getReason();
201 final State previous = this.state;
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": received session terminate reason="
206 + wrapper.reason
207 + "("
208 + Strings.nullToEmpty(wrapper.text)
209 + ") while in state "
210 + previous);
211 if (TERMINATED.contains(previous)) {
212 Log.d(
213 Config.LOGTAG,
214 id.account.getJid().asBareJid()
215 + ": ignoring session terminate because already in "
216 + previous);
217 return;
218 }
219 webRTCWrapper.close();
220 final State target = reasonToState(wrapper.reason);
221 transitionOrThrow(target);
222 writeLogMessage(target);
223 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 finish();
227 }
228
229 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
230 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
231 // INITIALIZED only after transport-info has been received
232 if (isInState(
233 State.NULL,
234 State.PROCEED,
235 State.SESSION_INITIALIZED,
236 State.SESSION_INITIALIZED_PRE_APPROVED,
237 State.SESSION_ACCEPTED)) {
238 final RtpContentMap contentMap;
239 try {
240 contentMap = RtpContentMap.of(jingle);
241 } catch (final IllegalArgumentException | NullPointerException e) {
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": improperly formatted contents; ignoring",
246 e);
247 respondOk(jinglePacket);
248 return;
249 }
250 receiveTransportInfo(jinglePacket, contentMap);
251 } else {
252 if (isTerminated()) {
253 respondOk(jinglePacket);
254 Log.d(
255 Config.LOGTAG,
256 id.account.getJid().asBareJid()
257 + ": ignoring out-of-order transport info; we where already terminated");
258 } else {
259 Log.d(
260 Config.LOGTAG,
261 id.account.getJid().asBareJid()
262 + ": received transport info while in state="
263 + this.state);
264 terminateWithOutOfOrder(jinglePacket);
265 }
266 }
267 }
268
269 private void receiveTransportInfo(
270 final Iq jinglePacket, final RtpContentMap contentMap) {
271 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
272 candidates = contentMap.contents.entrySet();
273 final RtpContentMap remote = getRemoteContentMap();
274 final Set<String> remoteContentIds =
275 remote == null ? Collections.emptySet() : remote.contents.keySet();
276 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
277 Log.d(
278 Config.LOGTAG,
279 "received transport-info for unknown contents "
280 + contentMap.contents.keySet()
281 + " (known: "
282 + remoteContentIds
283 + ")");
284 respondOk(jinglePacket);
285 pendingIceCandidates.addAll(candidates);
286 return;
287 }
288 if (this.state != State.SESSION_ACCEPTED) {
289 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
290 respondOk(jinglePacket);
291 pendingIceCandidates.addAll(candidates);
292 return;
293 }
294 // zero candidates + modified credentials are an ICE restart offer
295 if (checkForIceRestart(jinglePacket, contentMap)) {
296 return;
297 }
298 respondOk(jinglePacket);
299 try {
300 processCandidates(candidates);
301 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
302 Log.w(
303 Config.LOGTAG,
304 id.account.getJid().asBareJid()
305 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
306 }
307 }
308
309 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
310 final RtpContentMap modification;
311 try {
312 modification = RtpContentMap.of(jingle);
313 modification.requireContentDescriptions();
314 } catch (final RuntimeException e) {
315 Log.d(
316 Config.LOGTAG,
317 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
318 Throwables.getRootCause(e));
319 respondOk(iq);
320 webRTCWrapper.close();
321 sendSessionTerminate(Reason.of(e), e.getMessage());
322 return;
323 }
324 if (isInState(State.SESSION_ACCEPTED)) {
325 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
326 final ListenableFuture<RtpContentMap> future =
327 receiveRtpContentMap(
328 modification,
329 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
330 Futures.addCallback(
331 future,
332 new FutureCallback<>() {
333 @Override
334 public void onSuccess(final RtpContentMap rtpContentMap) {
335 receiveContentAdd(iq, rtpContentMap);
336 }
337
338 @Override
339 public void onFailure(@NonNull Throwable throwable) {
340 respondOk(iq);
341 final Throwable rootCause = Throwables.getRootCause(throwable);
342 Log.d(
343 Config.LOGTAG,
344 id.account.getJid().asBareJid()
345 + ": improperly formatted contents in content-add",
346 throwable);
347 webRTCWrapper.close();
348 sendSessionTerminate(
349 Reason.ofThrowable(rootCause), rootCause.getMessage());
350 }
351 },
352 MoreExecutors.directExecutor());
353 } else {
354 terminateWithOutOfOrder(iq);
355 }
356 }
357
358 private void receiveContentAdd(
359 final Iq jinglePacket, final RtpContentMap modification) {
360 final RtpContentMap remote = getRemoteContentMap();
361 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
362 respondOk(jinglePacket);
363 this.webRTCWrapper.close();
364 sendSessionTerminate(
365 Reason.FAILED_APPLICATION,
366 String.format(
367 "contents with names %s already exists",
368 Joiner.on(", ").join(modification.getNames())));
369 return;
370 }
371 final ContentAddition contentAddition =
372 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
373
374 final RtpContentMap outgoing = this.outgoingContentAdd;
375 final Set<ContentAddition.Summary> outgoingContentAddSummary =
376 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
377
378 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
379 if (isInitiator()) {
380 Log.d(
381 Config.LOGTAG,
382 id.getAccount().getJid().asBareJid()
383 + ": respond with tie break to matching content-add offer");
384 respondWithTieBreak(jinglePacket);
385 } else {
386 Log.d(
387 Config.LOGTAG,
388 id.getAccount().getJid().asBareJid()
389 + ": automatically accept matching content-add offer");
390 acceptContentAdd(contentAddition.summary, modification);
391 }
392 return;
393 }
394
395 // once we can display multiple video tracks we can be more loose with this condition
396 // theoretically it should also be fine to automatically accept audio only contents
397 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
398 Log.d(
399 Config.LOGTAG,
400 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
401 this.incomingContentAdd = modification;
402 respondOk(jinglePacket);
403 updateEndUserState();
404 } else {
405 respondOk(jinglePacket);
406 // TODO do we want to add a reason?
407 rejectContentAdd(modification);
408 }
409 }
410
411 private void receiveContentAccept(final Iq jinglePacket) {
412 final var jingle = jinglePacket.getExtension(Jingle.class);
413 final RtpContentMap receivedContentAccept;
414 try {
415 receivedContentAccept = RtpContentMap.of(jingle);
416 receivedContentAccept.requireContentDescriptions();
417 } catch (final RuntimeException e) {
418 Log.d(
419 Config.LOGTAG,
420 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
421 Throwables.getRootCause(e));
422 respondOk(jinglePacket);
423 webRTCWrapper.close();
424 sendSessionTerminate(Reason.of(e), e.getMessage());
425 return;
426 }
427
428 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
429 if (outgoingContentAdd == null) {
430 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
431 terminateWithOutOfOrder(jinglePacket);
432 return;
433 }
434 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
435 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
436 this.outgoingContentAdd = null;
437 respondOk(jinglePacket);
438 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
439 final ListenableFuture<RtpContentMap> future =
440 receiveRtpContentMap(
441 receivedContentAccept,
442 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
443 Futures.addCallback(
444 future,
445 new FutureCallback<>() {
446 @Override
447 public void onSuccess(final RtpContentMap result) {
448 receiveContentAccept(result);
449 }
450
451 @Override
452 public void onFailure(@NonNull final Throwable throwable) {
453 webRTCWrapper.close();
454 sendSessionTerminate(
455 Reason.ofThrowable(throwable), throwable.getMessage());
456 }
457 },
458 MoreExecutors.directExecutor());
459 } else {
460 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
461 terminateWithOutOfOrder(jinglePacket);
462 }
463 }
464
465 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
466 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
467 final RtpContentMap modifiedContentMap =
468 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
469
470 setRemoteContentMap(modifiedContentMap);
471
472 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
473
474 final org.webrtc.SessionDescription sdp =
475 new org.webrtc.SessionDescription(
476 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
477
478 try {
479 this.webRTCWrapper.setRemoteDescription(sdp).get();
480 } catch (final Exception e) {
481 final Throwable cause = Throwables.getRootCause(e);
482 Log.d(
483 Config.LOGTAG,
484 id.getAccount().getJid().asBareJid()
485 + ": unable to set remote description after receiving content-accept",
486 cause);
487 webRTCWrapper.close();
488 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
489 return;
490 }
491 Log.d(
492 Config.LOGTAG,
493 id.getAccount().getJid().asBareJid()
494 + ": remote has accepted content-add "
495 + ContentAddition.summary(receivedContentAccept));
496 processCandidates(receivedContentAccept.contents.entrySet());
497 updateEndUserState();
498 }
499
500 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
501 if (this.state != State.SESSION_ACCEPTED) {
502 terminateWithOutOfOrder(jinglePacket);
503 return;
504 }
505 final Map<String, Content.Senders> modification =
506 Maps.transformEntries(
507 jingle.getJingleContents(), (key, value) -> value.getSenders());
508 final boolean isInitiator = isInitiator();
509 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
510 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
511 final Set<String> currentOutgoingMediaIds =
512 currentOutgoing == null
513 ? Collections.emptySet()
514 : currentOutgoing.contents.keySet();
515 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
516 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
517 respondOk(jinglePacket);
518 final RtpContentMap modifiedContentMap;
519 try {
520 modifiedContentMap =
521 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
522 } catch (final IllegalArgumentException e) {
523 webRTCWrapper.close();
524 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
525 return;
526 }
527 this.outgoingContentAdd = modifiedContentMap;
528 Log.d(
529 Config.LOGTAG,
530 id.account.getJid().asBareJid()
531 + ": processed content-modification for pending content-add");
532 } else if (remoteContentMap != null
533 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
534 respondOk(jinglePacket);
535 final RtpContentMap modifiedRemoteContentMap;
536 try {
537 modifiedRemoteContentMap =
538 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
539 } catch (final IllegalArgumentException e) {
540 webRTCWrapper.close();
541 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
542 return;
543 }
544 final SessionDescription offer;
545 try {
546 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
547 } catch (final IllegalArgumentException | NullPointerException e) {
548 Log.d(
549 Config.LOGTAG,
550 id.getAccount().getJid().asBareJid()
551 + ": unable convert offer from content-modify to SDP",
552 e);
553 webRTCWrapper.close();
554 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
555 return;
556 }
557 Log.d(
558 Config.LOGTAG,
559 id.account.getJid().asBareJid() + ": auto accepting content-modification");
560 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
561 } else {
562 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
563 respondWithItemNotFound(jinglePacket);
564 }
565 }
566
567 private void autoAcceptContentModify(
568 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
569 this.setRemoteContentMap(modifiedRemoteContentMap);
570 final org.webrtc.SessionDescription sdp =
571 new org.webrtc.SessionDescription(
572 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
573 try {
574 this.webRTCWrapper.setRemoteDescription(sdp).get();
575 // auto accept is only done when we already have tracks
576 final SessionDescription answer = setLocalSessionDescription();
577 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
578 modifyLocalContentMap(rtpContentMap);
579 // we do not need to send an answer but do we have to resend the candidates currently in
580 // SDP?
581 // resendCandidatesFromSdp(answer);
582 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
583 } catch (final Exception e) {
584 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
585 webRTCWrapper.close();
586 sendSessionTerminate(Reason.FAILED_APPLICATION);
587 }
588 }
589
590 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
591 final SessionDescription answer) {
592 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
593 new ImmutableMultimap.Builder<>();
594 for (final SessionDescription.Media media : answer.media) {
595 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
596 if (Strings.isNullOrEmpty(mid)) {
597 continue;
598 }
599 for (final String sdpCandidate : media.attributes.get("candidate")) {
600 final IceUdpTransportInfo.Candidate candidate =
601 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
602 if (candidate != null) {
603 candidateBuilder.put(mid, candidate);
604 }
605 }
606 }
607 return candidateBuilder.build();
608 }
609
610 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
611 final RtpContentMap receivedContentReject;
612 try {
613 receivedContentReject = RtpContentMap.of(jingle);
614 } catch (final RuntimeException e) {
615 Log.d(
616 Config.LOGTAG,
617 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
618 Throwables.getRootCause(e));
619 respondOk(jinglePacket);
620 this.webRTCWrapper.close();
621 sendSessionTerminate(Reason.of(e), e.getMessage());
622 return;
623 }
624
625 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
626 if (outgoingContentAdd == null) {
627 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
628 terminateWithOutOfOrder(jinglePacket);
629 return;
630 }
631 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
632 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
633 this.outgoingContentAdd = null;
634 respondOk(jinglePacket);
635 Log.d(Config.LOGTAG, jinglePacket.toString());
636 receiveContentReject(ourSummary);
637 } else {
638 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
639 terminateWithOutOfOrder(jinglePacket);
640 }
641 }
642
643 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
644 try {
645 this.webRTCWrapper.removeTrack(Media.VIDEO);
646 final RtpContentMap localContentMap = customRollback();
647 modifyLocalContentMap(localContentMap);
648 } catch (final Exception e) {
649 final Throwable cause = Throwables.getRootCause(e);
650 Log.d(
651 Config.LOGTAG,
652 id.getAccount().getJid().asBareJid()
653 + ": unable to rollback local description after receiving content-reject",
654 cause);
655 webRTCWrapper.close();
656 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
657 return;
658 }
659 Log.d(
660 Config.LOGTAG,
661 id.getAccount().getJid().asBareJid()
662 + ": remote has rejected our content-add "
663 + summary);
664 }
665
666 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
667 final RtpContentMap receivedContentRemove;
668 try {
669 receivedContentRemove = RtpContentMap.of(jingle);
670 receivedContentRemove.requireContentDescriptions();
671 } catch (final RuntimeException e) {
672 Log.d(
673 Config.LOGTAG,
674 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
675 Throwables.getRootCause(e));
676 respondOk(jinglePacket);
677 this.webRTCWrapper.close();
678 sendSessionTerminate(Reason.of(e), e.getMessage());
679 return;
680 }
681 respondOk(jinglePacket);
682 receiveContentRemove(receivedContentRemove);
683 }
684
685 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
686 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
687 final Set<ContentAddition.Summary> contentAddSummary =
688 incomingContentAdd == null
689 ? Collections.emptySet()
690 : ContentAddition.summary(incomingContentAdd);
691 final Set<ContentAddition.Summary> removeSummary =
692 ContentAddition.summary(receivedContentRemove);
693 if (contentAddSummary.equals(removeSummary)) {
694 this.incomingContentAdd = null;
695 updateEndUserState();
696 } else {
697 webRTCWrapper.close();
698 sendSessionTerminate(
699 Reason.FAILED_APPLICATION,
700 String.format(
701 "%s only supports %s as a means to retract a not yet accepted %s",
702 BuildConfig.APP_NAME,
703 Jingle.Action.CONTENT_REMOVE,
704 Jingle.Action.CONTENT_ADD));
705 }
706 }
707
708 public synchronized void retractContentAdd() {
709 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
710 if (outgoingContentAdd == null) {
711 throw new IllegalStateException("Not outgoing content add");
712 }
713 try {
714 webRTCWrapper.removeTrack(Media.VIDEO);
715 final RtpContentMap localContentMap = customRollback();
716 modifyLocalContentMap(localContentMap);
717 } catch (final Exception e) {
718 final Throwable cause = Throwables.getRootCause(e);
719 Log.d(
720 Config.LOGTAG,
721 id.getAccount().getJid().asBareJid()
722 + ": unable to rollback local description after trying to retract content-add",
723 cause);
724 webRTCWrapper.close();
725 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
726 return;
727 }
728 this.outgoingContentAdd = null;
729 final Iq retract =
730 outgoingContentAdd
731 .toStub()
732 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
733 this.send(retract);
734 Log.d(
735 Config.LOGTAG,
736 id.getAccount().getJid()
737 + ": retract content-add "
738 + ContentAddition.summary(outgoingContentAdd));
739 }
740
741 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
742 final SessionDescription sdp = setLocalSessionDescription();
743 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
744 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
745 this.webRTCWrapper
746 .setRemoteDescription(
747 new org.webrtc.SessionDescription(
748 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
749 .get();
750 return localRtpContentMap;
751 }
752
753 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
754 final RtpContentMap currentRemote = getRemoteContentMap();
755 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
756 if (diff.isEmpty()) {
757 throw new IllegalStateException(
758 "Unexpected rollback condition. No difference between local and remote");
759 }
760 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
761 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
762 final RtpContentMap nextRemote =
763 currentRemote.addContent(
764 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
765 return SessionDescription.of(nextRemote, isResponder());
766 }
767 throw new IllegalStateException(
768 "Unexpected rollback condition. Senders were not uniformly none");
769 }
770
771 public synchronized void acceptContentAdd(
772 @NonNull final Set<ContentAddition.Summary> contentAddition) {
773 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
774 if (incomingContentAdd == null) {
775 throw new IllegalStateException("No incoming content add");
776 }
777
778 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
779 this.incomingContentAdd = null;
780 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
781 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
782 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
783 Log.d(
784 Config.LOGTAG,
785 "content addition is receive only. we want to upgrade to 'both'");
786 final RtpContentMap modifiedSenders =
787 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
788 final Iq proposedContentModification =
789 modifiedSenders
790 .toStub()
791 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
792 proposedContentModification.setTo(id.with);
793 xmppConnectionService.sendIqPacket(
794 id.account,
795 proposedContentModification,
796 (response) -> {
797 if (response.getType() == Iq.Type.RESULT) {
798 Log.d(
799 Config.LOGTAG,
800 id.account.getJid().asBareJid()
801 + ": remote has accepted our upgrade to senders=both");
802 acceptContentAdd(
803 ContentAddition.summary(modifiedSenders), modifiedSenders);
804 } else {
805 Log.d(
806 Config.LOGTAG,
807 id.account.getJid().asBareJid()
808 + ": remote has rejected our upgrade to senders=both");
809 acceptContentAdd(contentAddition, incomingContentAdd);
810 }
811 });
812 } else {
813 acceptContentAdd(contentAddition, incomingContentAdd);
814 }
815 } else {
816 throw new IllegalStateException(
817 "Accepted content add does not match pending content-add");
818 }
819 }
820
821 private void acceptContentAdd(
822 @NonNull final Set<ContentAddition.Summary> contentAddition,
823 final RtpContentMap incomingContentAdd) {
824 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
825 final RtpContentMap modifiedContentMap =
826 getRemoteContentMap().addContent(incomingContentAdd, setup);
827 this.setRemoteContentMap(modifiedContentMap);
828
829 final SessionDescription offer;
830 try {
831 offer = SessionDescription.of(modifiedContentMap, isResponder());
832 } catch (final IllegalArgumentException | NullPointerException e) {
833 Log.d(
834 Config.LOGTAG,
835 id.getAccount().getJid().asBareJid()
836 + ": unable convert offer from content-add to SDP",
837 e);
838 webRTCWrapper.close();
839 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
840 return;
841 }
842 this.incomingContentAdd = null;
843 acceptContentAdd(contentAddition, offer);
844 }
845
846 private void acceptContentAdd(
847 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
848 final org.webrtc.SessionDescription sdp =
849 new org.webrtc.SessionDescription(
850 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
851 try {
852 this.webRTCWrapper.setRemoteDescription(sdp).get();
853
854 // TODO add tracks for 'media' where contentAddition.senders matches
855
856 // TODO if senders.sending(isInitiator())
857
858 this.webRTCWrapper.addTrack(Media.VIDEO);
859
860 // TODO add additional transceivers for recv only cases
861
862 final SessionDescription answer = setLocalSessionDescription();
863 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
864
865 final RtpContentMap contentAcceptMap =
866 rtpContentMap.toContentModification(
867 Collections2.transform(contentAddition, ca -> ca.name));
868
869 Log.d(
870 Config.LOGTAG,
871 id.getAccount().getJid().asBareJid()
872 + ": sending content-accept "
873 + ContentAddition.summary(contentAcceptMap));
874
875 addIceCandidatesFromBlackLog();
876
877 modifyLocalContentMap(rtpContentMap);
878 final ListenableFuture<RtpContentMap> future =
879 prepareOutgoingContentMap(contentAcceptMap);
880 Futures.addCallback(
881 future,
882 new FutureCallback<>() {
883 @Override
884 public void onSuccess(final RtpContentMap rtpContentMap) {
885 sendContentAccept(rtpContentMap);
886 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
887 }
888
889 @Override
890 public void onFailure(@NonNull final Throwable throwable) {
891 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
892 }
893 },
894 MoreExecutors.directExecutor());
895 } catch (final Exception e) {
896 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
897 webRTCWrapper.close();
898 sendSessionTerminate(Reason.FAILED_APPLICATION);
899 }
900 }
901
902 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
903 final Iq iq =
904 contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
905 send(iq);
906 }
907
908 public synchronized void rejectContentAdd() {
909 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
910 if (incomingContentAdd == null) {
911 throw new IllegalStateException("No incoming content add");
912 }
913 this.incomingContentAdd = null;
914 updateEndUserState();
915 rejectContentAdd(incomingContentAdd);
916 }
917
918 private void rejectContentAdd(final RtpContentMap contentMap) {
919 final Iq iq =
920 contentMap
921 .toStub()
922 .toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
923 Log.d(
924 Config.LOGTAG,
925 id.getAccount().getJid().asBareJid()
926 + ": rejecting content "
927 + ContentAddition.summary(contentMap));
928 send(iq);
929 }
930
931 private boolean checkForIceRestart(
932 final Iq jinglePacket, final RtpContentMap rtpContentMap) {
933 final RtpContentMap existing = getRemoteContentMap();
934 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
935 final IceUdpTransportInfo.Credentials newCredentials;
936 try {
937 existingCredentials = existing.getCredentials();
938 newCredentials = rtpContentMap.getDistinctCredentials();
939 } catch (final IllegalStateException e) {
940 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
941 return false;
942 }
943 if (existingCredentials.contains(newCredentials)) {
944 return false;
945 }
946 // TODO an alternative approach is to check if we already got an iq result to our
947 // ICE-restart
948 // and if that's the case we are seeing an answer.
949 // This might be more spec compliant but also more error prone potentially
950 final boolean isSignalStateStable =
951 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
952 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
953 final boolean isOffer = rtpContentMap.emptyCandidates();
954 final RtpContentMap restartContentMap;
955 try {
956 if (isOffer) {
957 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
958 restartContentMap =
959 existing.modifiedCredentials(
960 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
961 } else {
962 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
963 Log.d(
964 Config.LOGTAG,
965 "received confirmation of ICE restart"
966 + newCredentials
967 + " peer_setup="
968 + setup);
969 // DTLS setup attribute needs to be rewritten to reflect current peer state
970 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
971 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
972 }
973 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
974 return isOffer;
975 } else {
976 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
977 respondWithTieBreak(jinglePacket);
978 return true;
979 }
980 } catch (final Exception exception) {
981 respondOk(jinglePacket);
982 final Throwable rootCause = Throwables.getRootCause(exception);
983 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
984 // If this happens a termination is already in progress
985 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
986 return true;
987 }
988 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
989 webRTCWrapper.close();
990 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
991 return true;
992 }
993 }
994
995 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
996 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
997 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
998 throw new IllegalStateException("Invalid peer setup");
999 }
1000 return peerSetup;
1001 }
1002
1003 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1004 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1005 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1006 }
1007 this.peerDtlsSetup = setup;
1008 }
1009
1010 private boolean applyIceRestart(
1011 final Iq jinglePacket,
1012 final RtpContentMap restartContentMap,
1013 final boolean isOffer)
1014 throws ExecutionException, InterruptedException {
1015 final SessionDescription sessionDescription =
1016 SessionDescription.of(restartContentMap, isResponder());
1017 final org.webrtc.SessionDescription.Type type =
1018 isOffer
1019 ? org.webrtc.SessionDescription.Type.OFFER
1020 : org.webrtc.SessionDescription.Type.ANSWER;
1021 org.webrtc.SessionDescription sdp =
1022 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1023 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1024 if (isInitiator()) {
1025 // We ignore the offer and respond with tie-break. This will clause the responder
1026 // not to apply the content map
1027 return false;
1028 }
1029 }
1030 webRTCWrapper.setRemoteDescription(sdp).get();
1031 setRemoteContentMap(restartContentMap);
1032 if (isOffer) {
1033 final SessionDescription localSessionDescription = setLocalSessionDescription();
1034 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1035 // We need to respond OK before sending any candidates
1036 respondOk(jinglePacket);
1037 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1038 } else {
1039 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1040 }
1041 return true;
1042 }
1043
1044 private void processCandidates(
1045 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1046 contents) {
1047 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1048 content : contents) {
1049 processCandidate(content);
1050 }
1051 }
1052
1053 private void processCandidate(
1054 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1055 content) {
1056 final RtpContentMap rtpContentMap = getRemoteContentMap();
1057 final List<String> indices = toIdentificationTags(rtpContentMap);
1058 final String sdpMid = content.getKey(); // aka content name
1059 final IceUdpTransportInfo transport = content.getValue().transport;
1060 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1061
1062 // TODO check that credentials remained the same
1063
1064 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1065 final String sdp;
1066 try {
1067 sdp = candidate.toSdpAttribute(credentials.ufrag);
1068 } catch (final IllegalArgumentException e) {
1069 Log.d(
1070 Config.LOGTAG,
1071 id.account.getJid().asBareJid()
1072 + ": ignoring invalid ICE candidate "
1073 + e.getMessage());
1074 continue;
1075 }
1076 final int mLineIndex = indices.indexOf(sdpMid);
1077 if (mLineIndex < 0) {
1078 Log.w(
1079 Config.LOGTAG,
1080 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1081 }
1082 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1083 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1084 this.webRTCWrapper.addIceCandidate(iceCandidate);
1085 }
1086 }
1087
1088 private RtpContentMap getRemoteContentMap() {
1089 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1090 }
1091
1092 private RtpContentMap getLocalContentMap() {
1093 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1094 }
1095
1096 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1097 final Group originalGroup = rtpContentMap.group;
1098 final List<String> identificationTags =
1099 originalGroup == null
1100 ? rtpContentMap.getNames()
1101 : originalGroup.getIdentificationTags();
1102 if (identificationTags.size() == 0) {
1103 Log.w(
1104 Config.LOGTAG,
1105 id.account.getJid().asBareJid()
1106 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1107 }
1108 return identificationTags;
1109 }
1110
1111 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1112 final Jingle jinglePacket, final boolean expectVerification) {
1113 try {
1114 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1115 } catch (final Exception e) {
1116 return Futures.immediateFailedFuture(e);
1117 }
1118 }
1119
1120 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1121 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1122 Log.d(
1123 Config.LOGTAG,
1124 "receiveRtpContentMap("
1125 + receivedContentMap.getClass().getSimpleName()
1126 + ",expectVerification="
1127 + expectVerification
1128 + ")");
1129 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1130 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1131 id.account
1132 .getAxolotlService()
1133 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1134 return Futures.transform(
1135 future,
1136 omemoVerifiedPayload -> {
1137 // TODO test if an exception here triggers a correct abort
1138 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1139 Log.d(
1140 Config.LOGTAG,
1141 id.account.getJid().asBareJid()
1142 + ": received verifiable DTLS fingerprint via "
1143 + omemoVerification);
1144 return omemoVerifiedPayload.getPayload();
1145 },
1146 MoreExecutors.directExecutor());
1147 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1148 return Futures.immediateFailedFuture(
1149 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1150 } else {
1151 return Futures.immediateFuture(receivedContentMap);
1152 }
1153 }
1154
1155 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1156 if (isInitiator()) {
1157 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1158 return;
1159 }
1160 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1161 Futures.addCallback(
1162 future,
1163 new FutureCallback<>() {
1164 @Override
1165 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1166 receiveSessionInitiate(jinglePacket, rtpContentMap);
1167 }
1168
1169 @Override
1170 public void onFailure(@NonNull final Throwable throwable) {
1171 respondOk(jinglePacket);
1172 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1173 }
1174 },
1175 MoreExecutors.directExecutor());
1176 }
1177
1178 private void receiveSessionInitiate(
1179 final Iq jinglePacket, final RtpContentMap contentMap) {
1180 try {
1181 contentMap.requireContentDescriptions();
1182 contentMap.requireDTLSFingerprint(true);
1183 } catch (final RuntimeException e) {
1184 Log.d(
1185 Config.LOGTAG,
1186 id.account.getJid().asBareJid() + ": improperly formatted contents",
1187 Throwables.getRootCause(e));
1188 respondOk(jinglePacket);
1189 sendSessionTerminate(Reason.of(e), e.getMessage());
1190 return;
1191 }
1192 Log.d(
1193 Config.LOGTAG,
1194 "processing session-init with " + contentMap.contents.size() + " contents");
1195 final State target;
1196 if (this.state == State.PROCEED) {
1197 Preconditions.checkState(
1198 proposedMedia != null && proposedMedia.size() > 0,
1199 "proposed media must be set when processing pre-approved session-initiate");
1200 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1201 sendSessionTerminate(
1202 Reason.SECURITY_ERROR,
1203 String.format(
1204 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1205 this.proposedMedia, contentMap.getMedia()));
1206 return;
1207 }
1208 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1209 } else {
1210 target = State.SESSION_INITIALIZED;
1211 setProposedMedia(contentMap.getMedia());
1212 }
1213 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1214 respondOk(jinglePacket);
1215 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1216 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1217 Log.d(
1218 Config.LOGTAG,
1219 id.account.getJid().asBareJid()
1220 + ": automatically accepting session-initiate");
1221 sendSessionAccept();
1222 } else {
1223 Log.d(
1224 Config.LOGTAG,
1225 id.account.getJid().asBareJid()
1226 + ": received not pre-approved session-initiate. start ringing");
1227 startRinging();
1228 }
1229 } else {
1230 Log.d(
1231 Config.LOGTAG,
1232 String.format(
1233 "%s: received session-initiate while in state %s",
1234 id.account.getJid().asBareJid(), state));
1235 terminateWithOutOfOrder(jinglePacket);
1236 }
1237 }
1238
1239 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1240 if (isResponder()) {
1241 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1242 return;
1243 }
1244 final ListenableFuture<RtpContentMap> future =
1245 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1246 Futures.addCallback(
1247 future,
1248 new FutureCallback<>() {
1249 @Override
1250 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1251 receiveSessionAccept(jinglePacket, rtpContentMap);
1252 }
1253
1254 @Override
1255 public void onFailure(@NonNull final Throwable throwable) {
1256 respondOk(jinglePacket);
1257 Log.d(
1258 Config.LOGTAG,
1259 id.account.getJid().asBareJid()
1260 + ": improperly formatted contents in session-accept",
1261 throwable);
1262 webRTCWrapper.close();
1263 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1264 }
1265 },
1266 MoreExecutors.directExecutor());
1267 }
1268
1269 private void receiveSessionAccept(
1270 final Iq jinglePacket, final RtpContentMap contentMap) {
1271 try {
1272 contentMap.requireContentDescriptions();
1273 contentMap.requireDTLSFingerprint();
1274 } catch (final RuntimeException e) {
1275 respondOk(jinglePacket);
1276 Log.d(
1277 Config.LOGTAG,
1278 id.account.getJid().asBareJid()
1279 + ": improperly formatted contents in session-accept",
1280 e);
1281 webRTCWrapper.close();
1282 sendSessionTerminate(Reason.of(e), e.getMessage());
1283 return;
1284 }
1285 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1286 if (!initiatorMedia.equals(contentMap.getMedia())) {
1287 sendSessionTerminate(
1288 Reason.SECURITY_ERROR,
1289 String.format(
1290 "Your session-included included media %s but our session-initiate was %s",
1291 this.proposedMedia, contentMap.getMedia()));
1292 return;
1293 }
1294 Log.d(
1295 Config.LOGTAG,
1296 "processing session-accept with " + contentMap.contents.size() + " contents");
1297 if (transition(State.SESSION_ACCEPTED)) {
1298 respondOk(jinglePacket);
1299 receiveSessionAccept(contentMap);
1300 } else {
1301 Log.d(
1302 Config.LOGTAG,
1303 String.format(
1304 "%s: received session-accept while in state %s",
1305 id.account.getJid().asBareJid(), state));
1306 respondOk(jinglePacket);
1307 }
1308 }
1309
1310 private void receiveSessionAccept(final RtpContentMap contentMap) {
1311 this.responderRtpContentMap = contentMap;
1312 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1313 final SessionDescription sessionDescription;
1314 try {
1315 sessionDescription = SessionDescription.of(contentMap, false);
1316 } catch (final IllegalArgumentException | NullPointerException e) {
1317 Log.d(
1318 Config.LOGTAG,
1319 id.account.getJid().asBareJid()
1320 + ": unable convert offer from session-accept to SDP",
1321 e);
1322 webRTCWrapper.close();
1323 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1324 return;
1325 }
1326 final org.webrtc.SessionDescription answer =
1327 new org.webrtc.SessionDescription(
1328 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1329 try {
1330 this.webRTCWrapper.setRemoteDescription(answer).get();
1331 } catch (final Exception e) {
1332 Log.d(
1333 Config.LOGTAG,
1334 id.account.getJid().asBareJid()
1335 + ": unable to set remote description after receiving session-accept",
1336 Throwables.getRootCause(e));
1337 webRTCWrapper.close();
1338 sendSessionTerminate(
1339 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1340 return;
1341 }
1342 processCandidates(contentMap.contents.entrySet());
1343 }
1344
1345 private void sendSessionAccept() {
1346 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1347 if (rtpContentMap == null) {
1348 throw new IllegalStateException("initiator RTP Content Map has not been set");
1349 }
1350 final SessionDescription offer;
1351 try {
1352 offer = SessionDescription.of(rtpContentMap, true);
1353 } catch (final IllegalArgumentException | NullPointerException e) {
1354 Log.d(
1355 Config.LOGTAG,
1356 id.account.getJid().asBareJid()
1357 + ": unable convert offer from session-initiate to SDP",
1358 e);
1359 webRTCWrapper.close();
1360 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1361 return;
1362 }
1363 sendSessionAccept(rtpContentMap.getMedia(), offer);
1364 }
1365
1366 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1367 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1368 }
1369
1370 private synchronized void sendSessionAccept(
1371 final Set<Media> media,
1372 final SessionDescription offer,
1373 final List<PeerConnection.IceServer> iceServers) {
1374 if (isTerminated()) {
1375 Log.w(
1376 Config.LOGTAG,
1377 id.account.getJid().asBareJid()
1378 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1379 return;
1380 }
1381 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1382 try {
1383 setupWebRTC(media, iceServers, !includeCandidates);
1384 } catch (final WebRTCWrapper.InitializationException e) {
1385 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1386 webRTCWrapper.close();
1387 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1388 return;
1389 }
1390 final org.webrtc.SessionDescription sdp =
1391 new org.webrtc.SessionDescription(
1392 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1393 try {
1394 this.webRTCWrapper.setRemoteDescription(sdp).get();
1395 addIceCandidatesFromBlackLog();
1396 org.webrtc.SessionDescription webRTCSessionDescription =
1397 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1398 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1399 } catch (final Exception e) {
1400 failureToAcceptSession(e);
1401 }
1402 }
1403
1404 private void failureToAcceptSession(final Throwable throwable) {
1405 if (isTerminated()) {
1406 return;
1407 }
1408 final Throwable rootCause = Throwables.getRootCause(throwable);
1409 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1410 webRTCWrapper.close();
1411 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1412 }
1413
1414 private void failureToPerformAction(
1415 final Jingle.Action action, final Throwable throwable) {
1416 if (isTerminated()) {
1417 return;
1418 }
1419 final Throwable rootCause = Throwables.getRootCause(throwable);
1420 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1421 webRTCWrapper.close();
1422 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1423 }
1424
1425 private void addIceCandidatesFromBlackLog() {
1426 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1427 while ((foo = this.pendingIceCandidates.poll()) != null) {
1428 processCandidate(foo);
1429 Log.d(
1430 Config.LOGTAG,
1431 id.account.getJid().asBareJid() + ": added candidate from back log");
1432 }
1433 }
1434
1435 private void prepareSessionAccept(
1436 final org.webrtc.SessionDescription webRTCSessionDescription,
1437 final boolean includeCandidates) {
1438 final SessionDescription sessionDescription =
1439 SessionDescription.parse(webRTCSessionDescription.description);
1440 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1441 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1442 if (includeCandidates) {
1443 candidates = parseCandidates(sessionDescription);
1444 } else {
1445 candidates = ImmutableMultimap.of();
1446 }
1447 this.responderRtpContentMap = respondingRtpContentMap;
1448 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1449 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1450 prepareOutgoingContentMap(respondingRtpContentMap);
1451 Futures.addCallback(
1452 outgoingContentMapFuture,
1453 new FutureCallback<>() {
1454 @Override
1455 public void onSuccess(final RtpContentMap outgoingContentMap) {
1456 if (includeCandidates) {
1457 Log.d(
1458 Config.LOGTAG,
1459 "including "
1460 + candidates.size()
1461 + " candidates in session accept");
1462 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1463 } else {
1464 sendSessionAccept(outgoingContentMap);
1465 }
1466 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1467 }
1468
1469 @Override
1470 public void onFailure(@NonNull Throwable throwable) {
1471 failureToAcceptSession(throwable);
1472 }
1473 },
1474 MoreExecutors.directExecutor());
1475 }
1476
1477 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1478 if (isTerminated()) {
1479 Log.w(
1480 Config.LOGTAG,
1481 id.account.getJid().asBareJid()
1482 + ": preparing session accept was too slow. already terminated. nothing to do.");
1483 return;
1484 }
1485 transitionOrThrow(State.SESSION_ACCEPTED);
1486 final Iq sessionAccept =
1487 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1488 send(sessionAccept);
1489 }
1490
1491 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1492 final RtpContentMap rtpContentMap) {
1493 if (this.omemoVerification.hasDeviceId()) {
1494 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1495 verifiedPayloadFuture =
1496 id.account
1497 .getAxolotlService()
1498 .encrypt(
1499 rtpContentMap,
1500 id.with,
1501 omemoVerification.getDeviceId());
1502 return Futures.transform(
1503 verifiedPayloadFuture,
1504 verifiedPayload -> {
1505 omemoVerification.setOrEnsureEqual(verifiedPayload);
1506 return verifiedPayload.getPayload();
1507 },
1508 MoreExecutors.directExecutor());
1509 } else {
1510 return Futures.immediateFuture(rtpContentMap);
1511 }
1512 }
1513
1514 synchronized void deliveryMessage(
1515 final Jid from,
1516 final Element message,
1517 final String serverMessageId,
1518 final long timestamp) {
1519 Log.d(
1520 Config.LOGTAG,
1521 id.account.getJid().asBareJid()
1522 + ": delivered message to JingleRtpConnection "
1523 + message);
1524 switch (message.getName()) {
1525 case "propose" -> receivePropose(
1526 from, Propose.upgrade(message), serverMessageId, timestamp);
1527 case "proceed" -> receiveProceed(
1528 from, Proceed.upgrade(message), serverMessageId, timestamp);
1529 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1530 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1531 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1532 }
1533 }
1534
1535 void deliverFailedProceed(final String message) {
1536 Log.d(
1537 Config.LOGTAG,
1538 id.account.getJid().asBareJid()
1539 + ": receive message error for proceed message ("
1540 + Strings.nullToEmpty(message)
1541 + ")");
1542 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1543 webRTCWrapper.close();
1544 Log.d(
1545 Config.LOGTAG,
1546 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1547 this.finish();
1548 }
1549 }
1550
1551 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1552 final boolean originatedFromMyself =
1553 from.asBareJid().equals(id.account.getJid().asBareJid());
1554 if (originatedFromMyself) {
1555 if (transition(State.ACCEPTED)) {
1556 acceptedOnOtherDevice(serverMsgId, timestamp);
1557 } else {
1558 Log.d(
1559 Config.LOGTAG,
1560 id.account.getJid().asBareJid()
1561 + ": unable to transition to accept because already in state="
1562 + this.state);
1563 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1564 }
1565 } else {
1566 Log.d(
1567 Config.LOGTAG,
1568 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1569 }
1570 }
1571
1572 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1573 if (serverMsgId != null) {
1574 this.message.setServerMsgId(serverMsgId);
1575 }
1576 this.message.setTime(timestamp);
1577 this.message.setCarbon(true); // indicate that call was accepted on other device
1578 this.writeLogMessageSuccess(0);
1579 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1580 this.finish();
1581 }
1582
1583 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1584 final boolean originatedFromMyself =
1585 from.asBareJid().equals(id.account.getJid().asBareJid());
1586 // reject from another one of my clients
1587 if (originatedFromMyself) {
1588 receiveRejectFromMyself(serverMsgId, timestamp);
1589 } else if (isInitiator()) {
1590 if (from.equals(id.with)) {
1591 receiveRejectFromResponder();
1592 } else {
1593 Log.d(
1594 Config.LOGTAG,
1595 id.account.getJid()
1596 + ": ignoring reject from "
1597 + from
1598 + " for session with "
1599 + id.with);
1600 }
1601 } else {
1602 Log.d(
1603 Config.LOGTAG,
1604 id.account.getJid()
1605 + ": ignoring reject from "
1606 + from
1607 + " for session with "
1608 + id.with);
1609 }
1610 }
1611
1612 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1613 if (transition(State.REJECTED)) {
1614 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1615 this.finish();
1616 if (serverMsgId != null) {
1617 this.message.setServerMsgId(serverMsgId);
1618 }
1619 this.message.setTime(timestamp);
1620 this.message.setCarbon(true); // indicate that call was rejected on other device
1621 writeLogMessageMissed();
1622 } else {
1623 Log.d(
1624 Config.LOGTAG,
1625 "not able to transition into REJECTED because already in " + this.state);
1626 }
1627 }
1628
1629 private void receiveRejectFromResponder() {
1630 if (isInState(State.PROCEED)) {
1631 Log.d(
1632 Config.LOGTAG,
1633 id.account.getJid()
1634 + ": received reject while still in proceed. callee reconsidered");
1635 closeTransitionLogFinish(State.REJECTED_RACED);
1636 return;
1637 }
1638 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1639 Log.d(
1640 Config.LOGTAG,
1641 id.account.getJid()
1642 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1643 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1644 return;
1645 }
1646 Log.d(
1647 Config.LOGTAG,
1648 id.account.getJid()
1649 + ": ignoring reject from responder because already in state "
1650 + this.state);
1651 }
1652
1653 private void receivePropose(
1654 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1655 final boolean originatedFromMyself =
1656 from.asBareJid().equals(id.account.getJid().asBareJid());
1657 if (originatedFromMyself) {
1658 Log.d(
1659 Config.LOGTAG,
1660 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1661 } else if (transition(
1662 State.PROPOSED,
1663 () -> {
1664 final Collection<RtpDescription> descriptions =
1665 Collections2.transform(
1666 Collections2.filter(
1667 propose.getDescriptions(),
1668 d -> d instanceof RtpDescription),
1669 input -> (RtpDescription) input);
1670 final Collection<Media> media =
1671 Collections2.transform(descriptions, RtpDescription::getMedia);
1672 Preconditions.checkState(
1673 !media.contains(Media.UNKNOWN),
1674 "RTP descriptions contain unknown media");
1675 Log.d(
1676 Config.LOGTAG,
1677 id.account.getJid().asBareJid()
1678 + ": received session proposal from "
1679 + from
1680 + " for "
1681 + media);
1682 this.setProposedMedia(Sets.newHashSet(media));
1683 })) {
1684 if (serverMsgId != null) {
1685 this.message.setServerMsgId(serverMsgId);
1686 }
1687 this.message.setTime(timestamp);
1688 startRinging();
1689 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1690 sendJingleMessage("ringing");
1691 }
1692 } else {
1693 Log.d(
1694 Config.LOGTAG,
1695 id.account.getJid()
1696 + ": ignoring session proposal because already in "
1697 + state);
1698 }
1699 }
1700
1701 private void startRinging() {
1702 this.callIntegration.setRinging();
1703 Log.d(
1704 Config.LOGTAG,
1705 id.account.getJid().asBareJid()
1706 + ": received call from "
1707 + id.with
1708 + ". start ringing");
1709 ringingTimeoutFuture =
1710 jingleConnectionManager.schedule(
1711 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1712 if (CallIntegration.selfManaged(xmppConnectionService)) {
1713 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1714 return;
1715 }
1716 }
1717 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1718 }
1719
1720 private synchronized void ringingTimeout() {
1721 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1722 switch (this.state) {
1723 case PROPOSED -> {
1724 message.markUnread();
1725 rejectCallFromProposed();
1726 }
1727 case SESSION_INITIALIZED -> {
1728 message.markUnread();
1729 rejectCallFromSessionInitiate();
1730 }
1731 }
1732 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1733 }
1734
1735 private void cancelRingingTimeout() {
1736 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1737 if (future != null && !future.isCancelled()) {
1738 future.cancel(false);
1739 }
1740 }
1741
1742 private void receiveProceed(
1743 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1744 final Set<Media> media =
1745 Preconditions.checkNotNull(
1746 this.proposedMedia, "Proposed media has to be set before handling proceed");
1747 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1748 if (from.equals(id.with)) {
1749 if (isInitiator()) {
1750 if (transition(State.PROCEED)) {
1751 if (serverMsgId != null) {
1752 this.message.setServerMsgId(serverMsgId);
1753 }
1754 this.message.setTime(timestamp);
1755 final Integer remoteDeviceId = proceed.getDeviceId();
1756 if (isOmemoEnabled()) {
1757 this.omemoVerification.setDeviceId(remoteDeviceId);
1758 } else {
1759 if (remoteDeviceId != null) {
1760 Log.d(
1761 Config.LOGTAG,
1762 id.account.getJid().asBareJid()
1763 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1764 }
1765 this.omemoVerification.setDeviceId(null);
1766 }
1767 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1768 } else {
1769 Log.d(
1770 Config.LOGTAG,
1771 String.format(
1772 "%s: ignoring proceed because already in %s",
1773 id.account.getJid().asBareJid(), this.state));
1774 }
1775 } else {
1776 Log.d(
1777 Config.LOGTAG,
1778 String.format(
1779 "%s: ignoring proceed because we were not initializing",
1780 id.account.getJid().asBareJid()));
1781 }
1782 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1783 if (transition(State.ACCEPTED)) {
1784 Log.d(
1785 Config.LOGTAG,
1786 id.account.getJid().asBareJid()
1787 + ": moved session with "
1788 + id.with
1789 + " into state accepted after received carbon copied proceed");
1790 acceptedOnOtherDevice(serverMsgId, timestamp);
1791 }
1792 } else {
1793 Log.d(
1794 Config.LOGTAG,
1795 String.format(
1796 "%s: ignoring proceed from %s. was expected from %s",
1797 id.account.getJid().asBareJid(), from, id.with));
1798 }
1799 }
1800
1801 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1802 if (from.equals(id.with)) {
1803 final State target =
1804 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1805 if (transition(target)) {
1806 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1807 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1808 Log.d(
1809 Config.LOGTAG,
1810 id.account.getJid().asBareJid()
1811 + ": session with "
1812 + id.with
1813 + " has been retracted (serverMsgId="
1814 + serverMsgId
1815 + ")");
1816 if (serverMsgId != null) {
1817 this.message.setServerMsgId(serverMsgId);
1818 }
1819 this.message.setTime(timestamp);
1820 if (target == State.RETRACTED) {
1821 this.message.markUnread();
1822 }
1823 writeLogMessageMissed();
1824 finish();
1825 } else {
1826 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1827 }
1828 } else {
1829 // TODO parse retract from self
1830 Log.d(
1831 Config.LOGTAG,
1832 id.account.getJid().asBareJid()
1833 + ": received retract from "
1834 + from
1835 + ". expected retract from"
1836 + id.with
1837 + ". ignoring");
1838 }
1839 }
1840
1841 public void sendSessionInitiate() {
1842 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1843 }
1844
1845 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1846 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1847 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1848 }
1849
1850 private synchronized void sendSessionInitiate(
1851 final Set<Media> media,
1852 final State targetState,
1853 final List<PeerConnection.IceServer> iceServers) {
1854 if (isTerminated()) {
1855 Log.w(
1856 Config.LOGTAG,
1857 id.account.getJid().asBareJid()
1858 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1859 return;
1860 }
1861 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1862 try {
1863 setupWebRTC(media, iceServers, !includeCandidates);
1864 } catch (final WebRTCWrapper.InitializationException e) {
1865 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1866 webRTCWrapper.close();
1867 sendRetract(Reason.ofThrowable(e));
1868 return;
1869 }
1870 try {
1871 org.webrtc.SessionDescription webRTCSessionDescription =
1872 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1873 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1874 } catch (final Exception e) {
1875 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1876 // exceptions
1877 failureToInitiateSession(e, targetState);
1878 }
1879 }
1880
1881 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1882 if (isTerminated()) {
1883 return;
1884 }
1885 Log.d(
1886 Config.LOGTAG,
1887 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1888 Throwables.getRootCause(throwable));
1889 webRTCWrapper.close();
1890 final Reason reason = Reason.ofThrowable(throwable);
1891 if (isInState(targetState)) {
1892 sendSessionTerminate(reason, throwable.getMessage());
1893 } else {
1894 sendRetract(reason);
1895 }
1896 }
1897
1898 private void sendRetract(final Reason reason) {
1899 // TODO embed reason into retract
1900 sendJingleMessage("retract", id.with.asBareJid());
1901 transitionOrThrow(reasonToState(reason));
1902 this.finish();
1903 }
1904
1905 private void prepareSessionInitiate(
1906 final org.webrtc.SessionDescription webRTCSessionDescription,
1907 final boolean includeCandidates,
1908 final State targetState) {
1909 final SessionDescription sessionDescription =
1910 SessionDescription.parse(webRTCSessionDescription.description);
1911 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1912 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1913 if (includeCandidates) {
1914 candidates = parseCandidates(sessionDescription);
1915 } else {
1916 candidates = ImmutableMultimap.of();
1917 }
1918 this.initiatorRtpContentMap = rtpContentMap;
1919 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1920 encryptSessionInitiate(rtpContentMap);
1921 Futures.addCallback(
1922 outgoingContentMapFuture,
1923 new FutureCallback<>() {
1924 @Override
1925 public void onSuccess(final RtpContentMap outgoingContentMap) {
1926 if (includeCandidates) {
1927 Log.d(
1928 Config.LOGTAG,
1929 "including "
1930 + candidates.size()
1931 + " candidates in session initiate");
1932 sendSessionInitiate(
1933 outgoingContentMap.withCandidates(candidates), targetState);
1934 } else {
1935 sendSessionInitiate(outgoingContentMap, targetState);
1936 }
1937 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1938 }
1939
1940 @Override
1941 public void onFailure(@NonNull final Throwable throwable) {
1942 failureToInitiateSession(throwable, targetState);
1943 }
1944 },
1945 MoreExecutors.directExecutor());
1946 }
1947
1948 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1949 if (isTerminated()) {
1950 Log.w(
1951 Config.LOGTAG,
1952 id.account.getJid().asBareJid()
1953 + ": preparing session was too slow. already terminated. nothing to do.");
1954 return;
1955 }
1956 this.transitionOrThrow(targetState);
1957 final Iq sessionInitiate =
1958 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1959 send(sessionInitiate);
1960 }
1961
1962 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1963 final RtpContentMap rtpContentMap) {
1964 if (this.omemoVerification.hasDeviceId()) {
1965 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1966 verifiedPayloadFuture =
1967 id.account
1968 .getAxolotlService()
1969 .encrypt(
1970 rtpContentMap,
1971 id.with,
1972 omemoVerification.getDeviceId());
1973 final ListenableFuture<RtpContentMap> future =
1974 Futures.transform(
1975 verifiedPayloadFuture,
1976 verifiedPayload -> {
1977 omemoVerification.setSessionFingerprint(
1978 verifiedPayload.getFingerprint());
1979 return verifiedPayload.getPayload();
1980 },
1981 MoreExecutors.directExecutor());
1982 if (Config.REQUIRE_RTP_VERIFICATION) {
1983 return future;
1984 }
1985 return Futures.catching(
1986 future,
1987 CryptoFailedException.class,
1988 e -> {
1989 Log.w(
1990 Config.LOGTAG,
1991 id.account.getJid().asBareJid()
1992 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1993 e);
1994 return rtpContentMap;
1995 },
1996 MoreExecutors.directExecutor());
1997 } else {
1998 return Futures.immediateFuture(rtpContentMap);
1999 }
2000 }
2001
2002 protected void sendSessionTerminate(final Reason reason) {
2003 sendSessionTerminate(reason, null);
2004 }
2005
2006 protected void sendSessionTerminate(final Reason reason, final String text) {
2007 sendSessionTerminate(reason, text, this::writeLogMessage);
2008 sendJingleMessageFinish(reason);
2009 }
2010
2011 private void sendTransportInfo(
2012 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2013 final RtpContentMap transportInfo;
2014 try {
2015 final RtpContentMap rtpContentMap =
2016 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2017 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2018 } catch (final Exception e) {
2019 Log.d(
2020 Config.LOGTAG,
2021 id.account.getJid().asBareJid()
2022 + ": unable to prepare transport-info from candidate for content="
2023 + contentName);
2024 return;
2025 }
2026 final Iq iq =
2027 transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2028 send(iq);
2029 }
2030
2031 public RtpEndUserState getEndUserState() {
2032 switch (this.state) {
2033 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2034 if (isInitiator()) {
2035 return RtpEndUserState.RINGING;
2036 } else {
2037 return RtpEndUserState.INCOMING_CALL;
2038 }
2039 }
2040 case PROCEED -> {
2041 if (isInitiator()) {
2042 return RtpEndUserState.RINGING;
2043 } else {
2044 return RtpEndUserState.ACCEPTING_CALL;
2045 }
2046 }
2047 case SESSION_INITIALIZED_PRE_APPROVED -> {
2048 if (isInitiator()) {
2049 return RtpEndUserState.RINGING;
2050 } else {
2051 return RtpEndUserState.CONNECTING;
2052 }
2053 }
2054 case SESSION_ACCEPTED -> {
2055 final ContentAddition ca = getPendingContentAddition();
2056 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2057 return RtpEndUserState.INCOMING_CONTENT_ADD;
2058 }
2059 return getPeerConnectionStateAsEndUserState();
2060 }
2061 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2062 if (isInitiator()) {
2063 return RtpEndUserState.DECLINED_OR_BUSY;
2064 } else {
2065 return RtpEndUserState.ENDED;
2066 }
2067 }
2068 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2069 return RtpEndUserState.ENDED;
2070 }
2071 case RETRACTED_RACED -> {
2072 if (isInitiator()) {
2073 return RtpEndUserState.ENDED;
2074 } else {
2075 return RtpEndUserState.RETRACTED;
2076 }
2077 }
2078 case TERMINATED_CONNECTIVITY_ERROR -> {
2079 return zeroDuration()
2080 ? RtpEndUserState.CONNECTIVITY_ERROR
2081 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2082 }
2083 case TERMINATED_APPLICATION_FAILURE -> {
2084 return RtpEndUserState.APPLICATION_ERROR;
2085 }
2086 case TERMINATED_SECURITY_ERROR -> {
2087 return RtpEndUserState.SECURITY_ERROR;
2088 }
2089 }
2090 throw new IllegalStateException(
2091 String.format("%s has no equivalent EndUserState", this.state));
2092 }
2093
2094 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2095 final PeerConnection.PeerConnectionState state;
2096 try {
2097 state = webRTCWrapper.getState();
2098 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2099 // We usually close the WebRTCWrapper *before* transitioning so we might still
2100 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2101 return RtpEndUserState.ENDING_CALL;
2102 }
2103 return switch (state) {
2104 case CONNECTED -> RtpEndUserState.CONNECTED;
2105 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2106 case CLOSED -> RtpEndUserState.ENDING_CALL;
2107 default -> zeroDuration()
2108 ? RtpEndUserState.CONNECTIVITY_ERROR
2109 : RtpEndUserState.RECONNECTING;
2110 };
2111 }
2112
2113 private boolean isPeerConnectionConnected() {
2114 try {
2115 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2116 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2117 return false;
2118 }
2119 }
2120
2121 private void updateCallIntegrationState() {
2122 switch (this.state) {
2123 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2124 if (isInitiator()) {
2125 this.callIntegration.setDialing();
2126 } else {
2127 this.callIntegration.setRinging();
2128 }
2129 }
2130 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2131 if (isInitiator()) {
2132 this.callIntegration.setDialing();
2133 } else {
2134 this.callIntegration.setInitialized();
2135 }
2136 }
2137 case SESSION_ACCEPTED -> {
2138 if (isPeerConnectionConnected()) {
2139 this.callIntegration.setActive();
2140 } else {
2141 this.callIntegration.setInitialized();
2142 }
2143 }
2144 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2145 if (isInitiator()) {
2146 this.callIntegration.busy();
2147 } else {
2148 this.callIntegration.rejected();
2149 }
2150 }
2151 case TERMINATED_SUCCESS -> this.callIntegration.success();
2152 case ACCEPTED -> this.callIntegration.accepted();
2153 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2154 .retracted();
2155 case TERMINATED_CONNECTIVITY_ERROR,
2156 TERMINATED_APPLICATION_FAILURE,
2157 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2158 default -> throw new IllegalStateException(
2159 String.format("%s is not handled", this.state));
2160 }
2161 }
2162
2163 public ContentAddition getPendingContentAddition() {
2164 final RtpContentMap in = this.incomingContentAdd;
2165 final RtpContentMap out = this.outgoingContentAdd;
2166 if (out != null) {
2167 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2168 } else if (in != null) {
2169 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2170 } else {
2171 return null;
2172 }
2173 }
2174
2175 public Set<Media> getMedia() {
2176 final State current = getState();
2177 if (current == State.NULL) {
2178 if (isInitiator()) {
2179 return Preconditions.checkNotNull(
2180 this.proposedMedia, "RTP connection has not been initialized properly");
2181 }
2182 throw new IllegalStateException("RTP connection has not been initialized yet");
2183 }
2184 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2185 return Preconditions.checkNotNull(
2186 this.proposedMedia, "RTP connection has not been initialized properly");
2187 }
2188 final RtpContentMap localContentMap = getLocalContentMap();
2189 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2190 if (localContentMap != null) {
2191 return localContentMap.getMedia();
2192 } else if (initiatorContentMap != null) {
2193 return initiatorContentMap.getMedia();
2194 } else if (isTerminated()) {
2195 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2196 } else {
2197 return Preconditions.checkNotNull(
2198 this.proposedMedia, "RTP connection has not been initialized properly");
2199 }
2200 }
2201
2202 public boolean isVerified() {
2203 final String fingerprint = this.omemoVerification.getFingerprint();
2204 if (fingerprint == null) {
2205 return false;
2206 }
2207 final FingerprintStatus status =
2208 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2209 return status != null && status.isVerified();
2210 }
2211
2212 public boolean addMedia(final Media media) {
2213 final Set<Media> currentMedia = getMedia();
2214 if (currentMedia.contains(media)) {
2215 throw new IllegalStateException(String.format("%s has already been proposed", media));
2216 }
2217 // TODO add state protection - can only add while ACCEPTED or so
2218 Log.d(Config.LOGTAG, "adding media: " + media);
2219 return webRTCWrapper.addTrack(media);
2220 }
2221
2222 public synchronized void acceptCall() {
2223 switch (this.state) {
2224 case PROPOSED -> {
2225 cancelRingingTimeout();
2226 acceptCallFromProposed();
2227 }
2228 case SESSION_INITIALIZED -> {
2229 cancelRingingTimeout();
2230 acceptCallFromSessionInitialized();
2231 }
2232 case ACCEPTED -> Log.w(
2233 Config.LOGTAG,
2234 id.account.getJid().asBareJid()
2235 + ": the call has already been accepted with another client. UI was just lagging behind");
2236 case PROCEED, SESSION_ACCEPTED -> Log.w(
2237 Config.LOGTAG,
2238 id.account.getJid().asBareJid()
2239 + ": the call has already been accepted. user probably double tapped the UI");
2240 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2241 }
2242 }
2243
2244 public synchronized void rejectCall() {
2245 if (isTerminated()) {
2246 Log.w(
2247 Config.LOGTAG,
2248 id.account.getJid().asBareJid()
2249 + ": received rejectCall() when session has already been terminated. nothing to do");
2250 return;
2251 }
2252 switch (this.state) {
2253 case PROPOSED -> rejectCallFromProposed();
2254 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2255 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2256 }
2257 }
2258
2259 public synchronized void integrationFailure() {
2260 final var state = getState();
2261 if (state == State.PROPOSED) {
2262 Log.e(
2263 Config.LOGTAG,
2264 id.account.getJid().asBareJid()
2265 + ": failed call integration in state proposed");
2266 rejectCallFromProposed();
2267 } else if (state == State.SESSION_INITIALIZED) {
2268 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2269 this.webRTCWrapper.close();
2270 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2271 } else {
2272 throw new IllegalStateException(
2273 String.format("Can not fail integration in state %s", state));
2274 }
2275 }
2276
2277 public synchronized void endCall() {
2278 if (isTerminated()) {
2279 Log.w(
2280 Config.LOGTAG,
2281 id.account.getJid().asBareJid()
2282 + ": received endCall() when session has already been terminated. nothing to do");
2283 return;
2284 }
2285 if (isInState(State.PROPOSED) && isResponder()) {
2286 rejectCallFromProposed();
2287 return;
2288 }
2289 if (isInState(State.PROCEED)) {
2290 if (isInitiator()) {
2291 retractFromProceed();
2292 } else {
2293 rejectCallFromProceed();
2294 }
2295 return;
2296 }
2297 if (isInitiator()
2298 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2299 this.webRTCWrapper.close();
2300 sendSessionTerminate(Reason.CANCEL);
2301 return;
2302 }
2303 if (isInState(State.SESSION_INITIALIZED)) {
2304 rejectCallFromSessionInitiate();
2305 return;
2306 }
2307 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2308 this.webRTCWrapper.close();
2309 sendSessionTerminate(Reason.SUCCESS);
2310 return;
2311 }
2312 if (isInState(
2313 State.TERMINATED_APPLICATION_FAILURE,
2314 State.TERMINATED_CONNECTIVITY_ERROR,
2315 State.TERMINATED_DECLINED_OR_BUSY)) {
2316 Log.d(
2317 Config.LOGTAG,
2318 "ignoring request to end call because already in state " + this.state);
2319 return;
2320 }
2321 throw new IllegalStateException(
2322 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2323 }
2324
2325 private void retractFromProceed() {
2326 Log.d(Config.LOGTAG, "retract from proceed");
2327 this.sendJingleMessage("retract");
2328 closeTransitionLogFinish(State.RETRACTED_RACED);
2329 }
2330
2331 private void closeTransitionLogFinish(final State state) {
2332 this.webRTCWrapper.close();
2333 transitionOrThrow(state);
2334 writeLogMessage(state);
2335 finish();
2336 }
2337
2338 private void setupWebRTC(
2339 final Set<Media> media,
2340 final List<PeerConnection.IceServer> iceServers,
2341 final boolean trickle)
2342 throws WebRTCWrapper.InitializationException {
2343 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2344 this.webRTCWrapper.setup(this.xmppConnectionService);
2345 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2346 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2347 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2348 }
2349
2350 private void acceptCallFromProposed() {
2351 transitionOrThrow(State.PROCEED);
2352 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2353 this.callIntegration.startAudioRouting();
2354 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2355 this.sendJingleMessage("proceed");
2356 }
2357
2358 private void rejectCallFromProposed() {
2359 transitionOrThrow(State.REJECTED);
2360 writeLogMessageMissed();
2361 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2362 this.sendJingleMessage("reject");
2363 finish();
2364 }
2365
2366 private void rejectCallFromProceed() {
2367 this.sendJingleMessage("reject");
2368 closeTransitionLogFinish(State.REJECTED_RACED);
2369 }
2370
2371 private void rejectCallFromSessionInitiate() {
2372 webRTCWrapper.close();
2373 sendSessionTerminate(Reason.DECLINE);
2374 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2375 }
2376
2377 private void sendJingleMessage(final String action) {
2378 sendJingleMessage(action, id.with);
2379 }
2380
2381 private void sendJingleMessage(final String action, final Jid to) {
2382 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2383 messagePacket.setType(im.conversations.android.xmpp.model.stanza.Message.Type.CHAT); // we want to carbon copy those
2384 messagePacket.setTo(to);
2385 final Element intent =
2386 messagePacket
2387 .addChild(action, Namespace.JINGLE_MESSAGE)
2388 .setAttribute("id", id.sessionId);
2389 if ("proceed".equals(action)) {
2390 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2391 if (isOmemoEnabled()) {
2392 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2393 final Element device =
2394 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2395 device.setAttribute("id", deviceId);
2396 }
2397 }
2398 messagePacket.addChild("store", "urn:xmpp:hints");
2399 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2400 }
2401
2402 private void sendJingleMessageFinish(final Reason reason) {
2403 final var account = id.getAccount();
2404 final var messagePacket =
2405 xmppConnectionService
2406 .getMessageGenerator()
2407 .sessionFinish(id.with, id.sessionId, reason);
2408 xmppConnectionService.sendMessagePacket(account, messagePacket);
2409 }
2410
2411 private boolean isOmemoEnabled() {
2412 final Conversational conversational = message.getConversation();
2413 if (conversational instanceof Conversation) {
2414 return ((Conversation) conversational).getNextEncryption()
2415 == Message.ENCRYPTION_AXOLOTL;
2416 }
2417 return false;
2418 }
2419
2420 private void acceptCallFromSessionInitialized() {
2421 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2422 this.callIntegration.startAudioRouting();
2423 sendSessionAccept();
2424 }
2425
2426 @Override
2427 protected synchronized boolean transition(final State target, final Runnable runnable) {
2428 if (super.transition(target, runnable)) {
2429 updateEndUserState();
2430 updateOngoingCallNotification();
2431 return true;
2432 } else {
2433 return false;
2434 }
2435 }
2436
2437 @Override
2438 public void onIceCandidate(final IceCandidate iceCandidate) {
2439 final RtpContentMap rtpContentMap =
2440 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2441 final IceUdpTransportInfo.Credentials credentials;
2442 try {
2443 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2444 } catch (final IllegalArgumentException e) {
2445 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2446 return;
2447 }
2448 final String uFrag = credentials.ufrag;
2449 final IceUdpTransportInfo.Candidate candidate =
2450 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2451 if (candidate == null) {
2452 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2453 return;
2454 }
2455 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2456 sendTransportInfo(iceCandidate.sdpMid, candidate);
2457 }
2458
2459 @Override
2460 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2461 Log.d(
2462 Config.LOGTAG,
2463 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2464 this.stateHistory.add(newState);
2465 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2466 this.sessionDuration.start();
2467 updateOngoingCallNotification();
2468 } else if (this.sessionDuration.isRunning()) {
2469 this.sessionDuration.stop();
2470 updateOngoingCallNotification();
2471 }
2472
2473 final boolean neverConnected =
2474 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2475
2476 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2477 if (neverConnected) {
2478 if (isTerminated()) {
2479 Log.d(
2480 Config.LOGTAG,
2481 id.account.getJid().asBareJid()
2482 + ": not sending session-terminate after connectivity error because session is already in state "
2483 + this.state);
2484 return;
2485 }
2486 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2487 return;
2488 } else {
2489 this.restartIce();
2490 }
2491 }
2492 updateEndUserState();
2493 }
2494
2495 private void restartIce() {
2496 this.stateHistory.clear();
2497 this.webRTCWrapper.restartIceAsync();
2498 }
2499
2500 @Override
2501 public void onRenegotiationNeeded() {
2502 this.webRTCWrapper.execute(this::renegotiate);
2503 }
2504
2505 private synchronized void renegotiate() {
2506 final SessionDescription sessionDescription;
2507 try {
2508 sessionDescription = setLocalSessionDescription();
2509 } catch (final Exception e) {
2510 final Throwable cause = Throwables.getRootCause(e);
2511 webRTCWrapper.close();
2512 if (isTerminated()) {
2513 Log.d(
2514 Config.LOGTAG,
2515 "failed to renegotiate. session was already terminated",
2516 cause);
2517 return;
2518 }
2519 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2520 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2521 return;
2522 }
2523 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2524 final RtpContentMap currentContentMap = getLocalContentMap();
2525 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2526 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2527
2528 Log.d(
2529 Config.LOGTAG,
2530 id.getAccount().getJid().asBareJid()
2531 + ": renegotiate. iceRestart="
2532 + iceRestart
2533 + " content id diff="
2534 + diff);
2535
2536 if (diff.hasModifications() && iceRestart) {
2537 webRTCWrapper.close();
2538 sendSessionTerminate(
2539 Reason.FAILED_APPLICATION,
2540 "WebRTC unexpectedly tried to modify content and transport at once");
2541 return;
2542 }
2543
2544 if (iceRestart) {
2545 initiateIceRestart(rtpContentMap);
2546 return;
2547 } else if (diff.isEmpty()) {
2548 Log.d(
2549 Config.LOGTAG,
2550 "renegotiation. nothing to do. SignalingState="
2551 + this.webRTCWrapper.getSignalingState());
2552 }
2553
2554 if (diff.added.isEmpty()) {
2555 return;
2556 }
2557 modifyLocalContentMap(rtpContentMap);
2558 sendContentAdd(rtpContentMap, diff.added);
2559 }
2560
2561 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2562 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2563 final Iq iq =
2564 transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2565 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2566 iq.setTo(id.with);
2567 xmppConnectionService.sendIqPacket(
2568 id.account,
2569 iq,
2570 (response) -> {
2571 if (response.getType() == Iq.Type.RESULT) {
2572 Log.d(Config.LOGTAG, "received success to our ice restart");
2573 setLocalContentMap(rtpContentMap);
2574 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2575 return;
2576 }
2577 if (response.getType() == Iq.Type.ERROR) {
2578 if (isTieBreak(response)) {
2579 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2580 return;
2581 }
2582 handleIqErrorResponse(response);
2583 }
2584 if (response.getType() == Iq.Type.TIMEOUT) {
2585 handleIqTimeoutResponse(response);
2586 }
2587 });
2588 }
2589
2590 private boolean isTieBreak(final Iq response) {
2591 final Element error = response.findChild("error");
2592 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2593 }
2594
2595 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2596 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2597 this.outgoingContentAdd = contentAdd;
2598 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2599 prepareOutgoingContentMap(contentAdd);
2600 Futures.addCallback(
2601 outgoingContentMapFuture,
2602 new FutureCallback<>() {
2603 @Override
2604 public void onSuccess(final RtpContentMap outgoingContentMap) {
2605 sendContentAdd(outgoingContentMap);
2606 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2607 }
2608
2609 @Override
2610 public void onFailure(@NonNull Throwable throwable) {
2611 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2612 }
2613 },
2614 MoreExecutors.directExecutor());
2615 }
2616
2617 private void sendContentAdd(final RtpContentMap contentAdd) {
2618
2619 final Iq iq =
2620 contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2621 iq.setTo(id.with);
2622 xmppConnectionService.sendIqPacket(
2623 id.account,
2624 iq,
2625 (response) -> {
2626 if (response.getType() == Iq.Type.RESULT) {
2627 Log.d(
2628 Config.LOGTAG,
2629 id.getAccount().getJid().asBareJid()
2630 + ": received ACK to our content-add");
2631 return;
2632 }
2633 if (response.getType() == Iq.Type.ERROR) {
2634 if (isTieBreak(response)) {
2635 this.outgoingContentAdd = null;
2636 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2637 return;
2638 }
2639 handleIqErrorResponse(response);
2640 }
2641 if (response.getType() == Iq.Type.TIMEOUT) {
2642 handleIqTimeoutResponse(response);
2643 }
2644 });
2645 }
2646
2647 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2648 if (isInitiator()) {
2649 this.initiatorRtpContentMap = rtpContentMap;
2650 } else {
2651 this.responderRtpContentMap = rtpContentMap;
2652 }
2653 }
2654
2655 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2656 if (isInitiator()) {
2657 this.responderRtpContentMap = rtpContentMap;
2658 } else {
2659 this.initiatorRtpContentMap = rtpContentMap;
2660 }
2661 }
2662
2663 // this method is to be used for content map modifications that modify media
2664 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2665 final RtpContentMap activeContents = rtpContentMap.activeContents();
2666 setLocalContentMap(activeContents);
2667 this.callIntegration.setAudioDeviceWhenAvailable(
2668 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2669 updateEndUserState();
2670 }
2671
2672 private SessionDescription setLocalSessionDescription()
2673 throws ExecutionException, InterruptedException {
2674 final org.webrtc.SessionDescription sessionDescription =
2675 this.webRTCWrapper.setLocalDescription(false).get();
2676 return SessionDescription.parse(sessionDescription.description);
2677 }
2678
2679 private void closeWebRTCSessionAfterFailedConnection() {
2680 this.webRTCWrapper.close();
2681 synchronized (this) {
2682 if (isTerminated()) {
2683 Log.d(
2684 Config.LOGTAG,
2685 id.account.getJid().asBareJid()
2686 + ": no need to send session-terminate after failed connection. Other party already did");
2687 return;
2688 }
2689 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2690 }
2691 }
2692
2693 public boolean zeroDuration() {
2694 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2695 }
2696
2697 public long getCallDuration() {
2698 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2699 }
2700
2701 @Override
2702 public CallIntegration getCallIntegration() {
2703 return this.callIntegration;
2704 }
2705
2706 public boolean isMicrophoneEnabled() {
2707 return webRTCWrapper.isMicrophoneEnabled();
2708 }
2709
2710 public boolean setMicrophoneEnabled(final boolean enabled) {
2711 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2712 }
2713
2714 public boolean isVideoEnabled() {
2715 return webRTCWrapper.isVideoEnabled();
2716 }
2717
2718 public void setVideoEnabled(final boolean enabled) {
2719 webRTCWrapper.setVideoEnabled(enabled);
2720 }
2721
2722 public boolean isCameraSwitchable() {
2723 return webRTCWrapper.isCameraSwitchable();
2724 }
2725
2726 public boolean isFrontCamera() {
2727 return webRTCWrapper.isFrontCamera();
2728 }
2729
2730 public ListenableFuture<Boolean> switchCamera() {
2731 return webRTCWrapper.switchCamera();
2732 }
2733
2734 @Override
2735 public synchronized void onCallIntegrationShowIncomingCallUi() {
2736 if (isTerminated()) {
2737 // there might be race conditions with the call integration service invoking this
2738 // callback when the rtp session has already ended.
2739 Log.w(
2740 Config.LOGTAG,
2741 "CallIntegration requested incoming call UI but session was already terminated");
2742 return;
2743 }
2744 // TODO apparently this can be called too early as well?
2745 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2746 }
2747
2748 @Override
2749 public void onCallIntegrationDisconnect() {
2750 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2751 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2752 rejectCall();
2753 } else {
2754 endCall();
2755 }
2756 }
2757
2758 @Override
2759 public void onCallIntegrationReject() {
2760 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2761 try {
2762 rejectCall();
2763 } catch (final IllegalStateException e) {
2764 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2765 }
2766 }
2767
2768 @Override
2769 public void onCallIntegrationAnswer() {
2770 // we need to start the UI to a) show it and b) be able to ask for permissions
2771 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2772 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2773 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2774 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2775 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2776 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2777 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2778 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2779 xmppConnectionService.startActivity(intent);
2780 }
2781
2782 @Override
2783 public void onCallIntegrationSilence() {
2784 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2785 }
2786
2787 @Override
2788 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2789 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2790 // that was previous disabled by the user. A proper implementation would probably be to
2791 // track user choice and enable the microphone with a userEnabled() &&
2792 // callIntegration.isMicrophoneEnabled() condition
2793 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2794 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2795 }
2796
2797 @Override
2798 public void onAudioDeviceChanged(
2799 final CallIntegration.AudioDevice selectedAudioDevice,
2800 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2801 Log.d(
2802 Config.LOGTAG,
2803 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2804 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2805 selectedAudioDevice, availableAudioDevices);
2806 }
2807
2808 private void updateEndUserState() {
2809 final RtpEndUserState endUserState = getEndUserState();
2810 this.updateCallIntegrationState();
2811 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2812 id.account, id.with, id.sessionId, endUserState);
2813 }
2814
2815 private void updateOngoingCallNotification() {
2816 final State state = this.state;
2817 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2818 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2819 && isResponder()) {
2820 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2821 xmppConnectionService.removeOngoingCall();
2822 return;
2823 }
2824 final boolean reconnecting;
2825 if (state == State.SESSION_ACCEPTED) {
2826 reconnecting =
2827 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2828 } else {
2829 reconnecting = false;
2830 }
2831 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2832 } else {
2833 xmppConnectionService.removeOngoingCall();
2834 }
2835 }
2836
2837 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2838 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2839 final Iq request = new Iq(Iq.Type.GET);
2840 request.setTo(id.account.getDomain());
2841 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2842 xmppConnectionService.sendIqPacket(
2843 id.account,
2844 request,
2845 (response) -> {
2846 final var iceServers = IceServers.parse(response);
2847 if (iceServers.isEmpty()) {
2848 Log.w(
2849 Config.LOGTAG,
2850 id.account.getJid().asBareJid()
2851 + ": no ICE server found "
2852 + response);
2853 }
2854 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2855 });
2856 } else {
2857 Log.w(
2858 Config.LOGTAG,
2859 id.account.getJid().asBareJid() + ": has no external service discovery");
2860 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2861 }
2862 }
2863
2864 @Override
2865 protected void terminateTransport() {
2866 this.webRTCWrapper.close();
2867 }
2868
2869 @Override
2870 protected void finish() {
2871 if (isTerminated()) {
2872 this.cancelRingingTimeout();
2873 this.callIntegration.verifyDisconnected();
2874 this.webRTCWrapper.verifyClosed();
2875 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2876 super.finish();
2877 try {
2878 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2879 log.getParentFile().mkdirs();
2880 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2881 } catch (final IOException e) { }
2882 } else {
2883 throw new IllegalStateException(
2884 String.format("Unable to call finish from %s", this.state));
2885 }
2886 }
2887
2888 private void writeLogMessage(final State state) {
2889 final long duration = getCallDuration();
2890 if (state == State.TERMINATED_SUCCESS
2891 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2892 writeLogMessageSuccess(duration);
2893 } else {
2894 writeLogMessageMissed();
2895 }
2896 }
2897
2898 private void writeLogMessageSuccess(final long duration) {
2899 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2900 this.writeMessage();
2901 }
2902
2903 private void writeLogMessageMissed() {
2904 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2905 this.writeMessage();
2906 }
2907
2908 private void writeMessage() {
2909 final Conversational conversational = message.getConversation();
2910 if (conversational instanceof Conversation) {
2911 ((Conversation) conversational).add(this.message);
2912 xmppConnectionService.createMessageAsync(message);
2913 xmppConnectionService.updateConversationUi();
2914 } else {
2915 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2916 }
2917 }
2918
2919 public Optional<VideoTrack> getLocalVideoTrack() {
2920 return webRTCWrapper.getLocalVideoTrack();
2921 }
2922
2923 public Optional<VideoTrack> getRemoteVideoTrack() {
2924 return webRTCWrapper.getRemoteVideoTrack();
2925 }
2926
2927 public EglBase.Context getEglBaseContext() {
2928 return webRTCWrapper.getEglBaseContext();
2929 }
2930
2931 void setProposedMedia(final Set<Media> media) {
2932 this.proposedMedia = media;
2933 this.callIntegration.setVideoState(
2934 Media.audioOnly(media)
2935 ? VideoProfile.STATE_AUDIO_ONLY
2936 : VideoProfile.STATE_BIDIRECTIONAL);
2937 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2938 }
2939
2940 public void fireStateUpdate() {
2941 final RtpEndUserState endUserState = getEndUserState();
2942 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2943 id.account, id.with, id.sessionId, endUserState);
2944 }
2945
2946 public boolean isSwitchToVideoAvailable() {
2947 final boolean prerequisite =
2948 Media.audioOnly(getMedia())
2949 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2950 .contains(getEndUserState());
2951 return prerequisite && remoteHasVideoFeature();
2952 }
2953
2954 private boolean remoteHasVideoFeature() {
2955 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2956 }
2957
2958 private boolean remoteHasSdpOfferAnswer() {
2959 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2960 }
2961
2962 @Override
2963 public Account getAccount() {
2964 return id.account;
2965 }
2966
2967 @Override
2968 public Jid getWith() {
2969 return id.with;
2970 }
2971
2972 @Override
2973 public String getSessionId() {
2974 return id.sessionId;
2975 }
2976
2977 private interface OnIceServersDiscovered {
2978 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2979 }
2980}