1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
51import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
52
53import im.conversations.android.xmpp.model.jingle.Jingle;
54import im.conversations.android.xmpp.model.stanza.Iq;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 final var contact = id.getContact();
122 this.callIntegration.setCallerDisplayName(
123 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
124 this.callIntegration.setInitialized();
125 }
126
127 JingleRtpConnection(
128 final JingleConnectionManager jingleConnectionManager,
129 final Id id,
130 final Jid initiator,
131 final CallIntegration callIntegration) {
132 super(jingleConnectionManager, id, initiator);
133 final Conversation conversation =
134 jingleConnectionManager
135 .getXmppConnectionService()
136 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
137 this.message =
138 new Message(
139 conversation,
140 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
141 Message.TYPE_RTP_SESSION,
142 id.sessionId);
143 this.callIntegration = callIntegration;
144 this.callIntegration.setCallback(this);
145 }
146
147 @Override
148 synchronized void deliverPacket(final Iq iq) {
149 final var jingle = iq.getExtension(Jingle.class);
150 switch (jingle.getAction()) {
151 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
152 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
153 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
154 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
155 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
156 case CONTENT_ACCEPT -> receiveContentAccept(iq);
157 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
158 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
159 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
160 default -> {
161 respondOk(iq);
162 Log.d(
163 Config.LOGTAG,
164 String.format(
165 "%s: received unhandled jingle action %s",
166 id.account.getJid().asBareJid(), jingle.getAction()));
167 }
168 }
169 }
170
171 @Override
172 synchronized void notifyRebound() {
173 if (isTerminated()) {
174 return;
175 }
176 webRTCWrapper.close();
177 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
178 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
179 }
180 if (isInState(
181 State.SESSION_INITIALIZED,
182 State.SESSION_INITIALIZED_PRE_APPROVED,
183 State.SESSION_ACCEPTED)) {
184 // we might have already changed resources (full jid) at this point; so this might not
185 // even reach the other party
186 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
187 } else {
188 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
189 finish();
190 }
191 }
192
193 public boolean applyDtmfTone(final String tone) {
194 return webRTCWrapper.applyDtmfTone(tone);
195 }
196
197 private void receiveSessionTerminate(final Iq jinglePacket) {
198 respondOk(jinglePacket);
199 final var jingle = jinglePacket.getExtension(Jingle.class);
200 final Jingle.ReasonWrapper wrapper = jingle.getReason();
201 final State previous = this.state;
202 Log.d(
203 Config.LOGTAG,
204 id.account.getJid().asBareJid()
205 + ": received session terminate reason="
206 + wrapper.reason
207 + "("
208 + Strings.nullToEmpty(wrapper.text)
209 + ") while in state "
210 + previous);
211 if (TERMINATED.contains(previous)) {
212 Log.d(
213 Config.LOGTAG,
214 id.account.getJid().asBareJid()
215 + ": ignoring session terminate because already in "
216 + previous);
217 return;
218 }
219 webRTCWrapper.close();
220 final State target = reasonToState(wrapper.reason);
221 transitionOrThrow(target);
222 writeLogMessage(target);
223 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 finish();
227 }
228
229 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
230 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
231 // INITIALIZED only after transport-info has been received
232 if (isInState(
233 State.NULL,
234 State.PROCEED,
235 State.SESSION_INITIALIZED,
236 State.SESSION_INITIALIZED_PRE_APPROVED,
237 State.SESSION_ACCEPTED)) {
238 final RtpContentMap contentMap;
239 try {
240 contentMap = RtpContentMap.of(jingle);
241 } catch (final IllegalArgumentException | NullPointerException e) {
242 Log.d(
243 Config.LOGTAG,
244 id.account.getJid().asBareJid()
245 + ": improperly formatted contents; ignoring",
246 e);
247 respondOk(jinglePacket);
248 return;
249 }
250 receiveTransportInfo(jinglePacket, contentMap);
251 } else {
252 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
253 }
254 }
255
256 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
257 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
258 candidates = contentMap.contents.entrySet();
259 final RtpContentMap remote = getRemoteContentMap();
260 final Set<String> remoteContentIds =
261 remote == null ? Collections.emptySet() : remote.contents.keySet();
262 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
263 Log.d(
264 Config.LOGTAG,
265 "received transport-info for unknown contents "
266 + contentMap.contents.keySet()
267 + " (known: "
268 + remoteContentIds
269 + ")");
270 respondOk(jinglePacket);
271 pendingIceCandidates.addAll(candidates);
272 return;
273 }
274 if (this.state != State.SESSION_ACCEPTED) {
275 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
276 respondOk(jinglePacket);
277 pendingIceCandidates.addAll(candidates);
278 return;
279 }
280 // zero candidates + modified credentials are an ICE restart offer
281 if (checkForIceRestart(jinglePacket, contentMap)) {
282 return;
283 }
284 respondOk(jinglePacket);
285 try {
286 processCandidates(candidates);
287 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
288 Log.w(
289 Config.LOGTAG,
290 id.account.getJid().asBareJid()
291 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
292 }
293 }
294
295 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
296 final RtpContentMap modification;
297 try {
298 modification = RtpContentMap.of(jingle);
299 modification.requireContentDescriptions();
300 } catch (final RuntimeException e) {
301 Log.d(
302 Config.LOGTAG,
303 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
304 Throwables.getRootCause(e));
305 respondOk(iq);
306 webRTCWrapper.close();
307 sendSessionTerminate(Reason.of(e), e.getMessage());
308 return;
309 }
310 if (isInState(State.SESSION_ACCEPTED)) {
311 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
312 final ListenableFuture<RtpContentMap> future =
313 receiveRtpContentMap(
314 modification,
315 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
316 Futures.addCallback(
317 future,
318 new FutureCallback<>() {
319 @Override
320 public void onSuccess(final RtpContentMap rtpContentMap) {
321 receiveContentAdd(iq, rtpContentMap);
322 }
323
324 @Override
325 public void onFailure(@NonNull Throwable throwable) {
326 respondOk(iq);
327 final Throwable rootCause = Throwables.getRootCause(throwable);
328 Log.d(
329 Config.LOGTAG,
330 id.account.getJid().asBareJid()
331 + ": improperly formatted contents in content-add",
332 throwable);
333 webRTCWrapper.close();
334 sendSessionTerminate(
335 Reason.ofThrowable(rootCause), rootCause.getMessage());
336 }
337 },
338 MoreExecutors.directExecutor());
339 } else {
340 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
341 }
342 }
343
344 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
345 final RtpContentMap remote = getRemoteContentMap();
346 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
347 respondOk(jinglePacket);
348 this.webRTCWrapper.close();
349 sendSessionTerminate(
350 Reason.FAILED_APPLICATION,
351 String.format(
352 "contents with names %s already exists",
353 Joiner.on(", ").join(modification.getNames())));
354 return;
355 }
356 final ContentAddition contentAddition =
357 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
358
359 final RtpContentMap outgoing = this.outgoingContentAdd;
360 final Set<ContentAddition.Summary> outgoingContentAddSummary =
361 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
362
363 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
364 if (isInitiator()) {
365 Log.d(
366 Config.LOGTAG,
367 id.getAccount().getJid().asBareJid()
368 + ": respond with tie break to matching content-add offer");
369 respondWithTieBreak(jinglePacket);
370 } else {
371 Log.d(
372 Config.LOGTAG,
373 id.getAccount().getJid().asBareJid()
374 + ": automatically accept matching content-add offer");
375 acceptContentAdd(contentAddition.summary, modification);
376 }
377 return;
378 }
379
380 // once we can display multiple video tracks we can be more loose with this condition
381 // theoretically it should also be fine to automatically accept audio only contents
382 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
383 Log.d(
384 Config.LOGTAG,
385 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
386 this.incomingContentAdd = modification;
387 respondOk(jinglePacket);
388 updateEndUserState();
389 } else {
390 respondOk(jinglePacket);
391 // TODO do we want to add a reason?
392 rejectContentAdd(modification);
393 }
394 }
395
396 private void receiveContentAccept(final Iq jinglePacket) {
397 final var jingle = jinglePacket.getExtension(Jingle.class);
398 final RtpContentMap receivedContentAccept;
399 try {
400 receivedContentAccept = RtpContentMap.of(jingle);
401 receivedContentAccept.requireContentDescriptions();
402 } catch (final RuntimeException e) {
403 Log.d(
404 Config.LOGTAG,
405 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
406 Throwables.getRootCause(e));
407 respondOk(jinglePacket);
408 webRTCWrapper.close();
409 sendSessionTerminate(Reason.of(e), e.getMessage());
410 return;
411 }
412
413 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
414 if (outgoingContentAdd == null) {
415 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
416 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
417 return;
418 }
419 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
420 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
421 this.outgoingContentAdd = null;
422 respondOk(jinglePacket);
423 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
424 final ListenableFuture<RtpContentMap> future =
425 receiveRtpContentMap(
426 receivedContentAccept,
427 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
428 Futures.addCallback(
429 future,
430 new FutureCallback<>() {
431 @Override
432 public void onSuccess(final RtpContentMap result) {
433 receiveContentAccept(result);
434 }
435
436 @Override
437 public void onFailure(@NonNull final Throwable throwable) {
438 webRTCWrapper.close();
439 sendSessionTerminate(
440 Reason.ofThrowable(throwable), throwable.getMessage());
441 }
442 },
443 MoreExecutors.directExecutor());
444 } else {
445 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
446 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
447 }
448 }
449
450 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
451 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
452 final RtpContentMap modifiedContentMap =
453 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
454
455 setRemoteContentMap(modifiedContentMap);
456
457 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
458
459 final org.webrtc.SessionDescription sdp =
460 new org.webrtc.SessionDescription(
461 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
462
463 try {
464 this.webRTCWrapper.setRemoteDescription(sdp).get();
465 } catch (final Exception e) {
466 final Throwable cause = Throwables.getRootCause(e);
467 Log.d(
468 Config.LOGTAG,
469 id.getAccount().getJid().asBareJid()
470 + ": unable to set remote description after receiving content-accept",
471 cause);
472 webRTCWrapper.close();
473 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
474 return;
475 }
476 Log.d(
477 Config.LOGTAG,
478 id.getAccount().getJid().asBareJid()
479 + ": remote has accepted content-add "
480 + ContentAddition.summary(receivedContentAccept));
481 processCandidates(receivedContentAccept.contents.entrySet());
482 updateEndUserState();
483 }
484
485 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
486 if (this.state != State.SESSION_ACCEPTED) {
487 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
488 return;
489 }
490 final Map<String, Content.Senders> modification =
491 Maps.transformEntries(
492 jingle.getJingleContents(), (key, value) -> value.getSenders());
493 final boolean isInitiator = isInitiator();
494 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
495 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
496 final Set<String> currentOutgoingMediaIds =
497 currentOutgoing == null
498 ? Collections.emptySet()
499 : currentOutgoing.contents.keySet();
500 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
501 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
502 respondOk(jinglePacket);
503 final RtpContentMap modifiedContentMap;
504 try {
505 modifiedContentMap =
506 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
507 } catch (final IllegalArgumentException e) {
508 webRTCWrapper.close();
509 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
510 return;
511 }
512 this.outgoingContentAdd = modifiedContentMap;
513 Log.d(
514 Config.LOGTAG,
515 id.account.getJid().asBareJid()
516 + ": processed content-modification for pending content-add");
517 } else if (remoteContentMap != null
518 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
519 respondOk(jinglePacket);
520 final RtpContentMap modifiedRemoteContentMap;
521 try {
522 modifiedRemoteContentMap =
523 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
524 } catch (final IllegalArgumentException e) {
525 webRTCWrapper.close();
526 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
527 return;
528 }
529 final SessionDescription offer;
530 try {
531 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
532 } catch (final IllegalArgumentException | NullPointerException e) {
533 Log.d(
534 Config.LOGTAG,
535 id.getAccount().getJid().asBareJid()
536 + ": unable convert offer from content-modify to SDP",
537 e);
538 webRTCWrapper.close();
539 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
540 return;
541 }
542 Log.d(
543 Config.LOGTAG,
544 id.account.getJid().asBareJid() + ": auto accepting content-modification");
545 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
546 } else {
547 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
548 respondWithItemNotFound(jinglePacket);
549 }
550 }
551
552 private void autoAcceptContentModify(
553 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
554 this.setRemoteContentMap(modifiedRemoteContentMap);
555 final org.webrtc.SessionDescription sdp =
556 new org.webrtc.SessionDescription(
557 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
558 try {
559 this.webRTCWrapper.setRemoteDescription(sdp).get();
560 // auto accept is only done when we already have tracks
561 final SessionDescription answer = setLocalSessionDescription();
562 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
563 modifyLocalContentMap(rtpContentMap);
564 // we do not need to send an answer but do we have to resend the candidates currently in
565 // SDP?
566 // resendCandidatesFromSdp(answer);
567 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
568 } catch (final Exception e) {
569 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
570 webRTCWrapper.close();
571 sendSessionTerminate(Reason.FAILED_APPLICATION);
572 }
573 }
574
575 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
576 final SessionDescription answer) {
577 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
578 new ImmutableMultimap.Builder<>();
579 for (final SessionDescription.Media media : answer.media) {
580 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
581 if (Strings.isNullOrEmpty(mid)) {
582 continue;
583 }
584 for (final String sdpCandidate : media.attributes.get("candidate")) {
585 final IceUdpTransportInfo.Candidate candidate =
586 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
587 if (candidate != null) {
588 candidateBuilder.put(mid, candidate);
589 }
590 }
591 }
592 return candidateBuilder.build();
593 }
594
595 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
596 final RtpContentMap receivedContentReject;
597 try {
598 receivedContentReject = RtpContentMap.of(jingle);
599 } catch (final RuntimeException e) {
600 Log.d(
601 Config.LOGTAG,
602 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
603 Throwables.getRootCause(e));
604 respondOk(jinglePacket);
605 this.webRTCWrapper.close();
606 sendSessionTerminate(Reason.of(e), e.getMessage());
607 return;
608 }
609
610 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
611 if (outgoingContentAdd == null) {
612 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
613 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
614 return;
615 }
616 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
617 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
618 this.outgoingContentAdd = null;
619 respondOk(jinglePacket);
620 Log.d(Config.LOGTAG, jinglePacket.toString());
621 receiveContentReject(ourSummary);
622 } else {
623 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
624 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
625 }
626 }
627
628 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
629 try {
630 this.webRTCWrapper.removeTrack(Media.VIDEO);
631 final RtpContentMap localContentMap = customRollback();
632 modifyLocalContentMap(localContentMap);
633 } catch (final Exception e) {
634 final Throwable cause = Throwables.getRootCause(e);
635 Log.d(
636 Config.LOGTAG,
637 id.getAccount().getJid().asBareJid()
638 + ": unable to rollback local description after receiving content-reject",
639 cause);
640 webRTCWrapper.close();
641 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
642 return;
643 }
644 Log.d(
645 Config.LOGTAG,
646 id.getAccount().getJid().asBareJid()
647 + ": remote has rejected our content-add "
648 + summary);
649 }
650
651 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
652 final RtpContentMap receivedContentRemove;
653 try {
654 receivedContentRemove = RtpContentMap.of(jingle);
655 receivedContentRemove.requireContentDescriptions();
656 } catch (final RuntimeException e) {
657 Log.d(
658 Config.LOGTAG,
659 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
660 Throwables.getRootCause(e));
661 respondOk(jinglePacket);
662 this.webRTCWrapper.close();
663 sendSessionTerminate(Reason.of(e), e.getMessage());
664 return;
665 }
666 respondOk(jinglePacket);
667 receiveContentRemove(receivedContentRemove);
668 }
669
670 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
671 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
672 final Set<ContentAddition.Summary> contentAddSummary =
673 incomingContentAdd == null
674 ? Collections.emptySet()
675 : ContentAddition.summary(incomingContentAdd);
676 final Set<ContentAddition.Summary> removeSummary =
677 ContentAddition.summary(receivedContentRemove);
678 if (contentAddSummary.equals(removeSummary)) {
679 this.incomingContentAdd = null;
680 updateEndUserState();
681 } else {
682 webRTCWrapper.close();
683 sendSessionTerminate(
684 Reason.FAILED_APPLICATION,
685 String.format(
686 "%s only supports %s as a means to retract a not yet accepted %s",
687 BuildConfig.APP_NAME,
688 Jingle.Action.CONTENT_REMOVE,
689 Jingle.Action.CONTENT_ADD));
690 }
691 }
692
693 public synchronized void retractContentAdd() {
694 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
695 if (outgoingContentAdd == null) {
696 throw new IllegalStateException("Not outgoing content add");
697 }
698 try {
699 webRTCWrapper.removeTrack(Media.VIDEO);
700 final RtpContentMap localContentMap = customRollback();
701 modifyLocalContentMap(localContentMap);
702 } catch (final Exception e) {
703 final Throwable cause = Throwables.getRootCause(e);
704 Log.d(
705 Config.LOGTAG,
706 id.getAccount().getJid().asBareJid()
707 + ": unable to rollback local description after trying to retract content-add",
708 cause);
709 webRTCWrapper.close();
710 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
711 return;
712 }
713 this.outgoingContentAdd = null;
714 final Iq retract =
715 outgoingContentAdd
716 .toStub()
717 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
718 this.send(retract);
719 Log.d(
720 Config.LOGTAG,
721 id.getAccount().getJid()
722 + ": retract content-add "
723 + ContentAddition.summary(outgoingContentAdd));
724 }
725
726 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
727 final SessionDescription sdp = setLocalSessionDescription();
728 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
729 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
730 this.webRTCWrapper
731 .setRemoteDescription(
732 new org.webrtc.SessionDescription(
733 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
734 .get();
735 return localRtpContentMap;
736 }
737
738 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
739 final RtpContentMap currentRemote = getRemoteContentMap();
740 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
741 if (diff.isEmpty()) {
742 throw new IllegalStateException(
743 "Unexpected rollback condition. No difference between local and remote");
744 }
745 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
746 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
747 final RtpContentMap nextRemote =
748 currentRemote.addContent(
749 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
750 return SessionDescription.of(nextRemote, isResponder());
751 }
752 throw new IllegalStateException(
753 "Unexpected rollback condition. Senders were not uniformly none");
754 }
755
756 public synchronized void acceptContentAdd(
757 @NonNull final Set<ContentAddition.Summary> contentAddition) {
758 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
759 if (incomingContentAdd == null) {
760 throw new IllegalStateException("No incoming content add");
761 }
762
763 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
764 this.incomingContentAdd = null;
765 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
766 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
767 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
768 Log.d(
769 Config.LOGTAG,
770 "content addition is receive only. we want to upgrade to 'both'");
771 final RtpContentMap modifiedSenders =
772 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
773 final Iq proposedContentModification =
774 modifiedSenders
775 .toStub()
776 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
777 proposedContentModification.setTo(id.with);
778 xmppConnectionService.sendIqPacket(
779 id.account,
780 proposedContentModification,
781 (response) -> {
782 if (response.getType() == Iq.Type.RESULT) {
783 Log.d(
784 Config.LOGTAG,
785 id.account.getJid().asBareJid()
786 + ": remote has accepted our upgrade to senders=both");
787 acceptContentAdd(
788 ContentAddition.summary(modifiedSenders), modifiedSenders);
789 } else {
790 Log.d(
791 Config.LOGTAG,
792 id.account.getJid().asBareJid()
793 + ": remote has rejected our upgrade to senders=both");
794 acceptContentAdd(contentAddition, incomingContentAdd);
795 }
796 });
797 } else {
798 acceptContentAdd(contentAddition, incomingContentAdd);
799 }
800 } else {
801 throw new IllegalStateException(
802 "Accepted content add does not match pending content-add");
803 }
804 }
805
806 private void acceptContentAdd(
807 @NonNull final Set<ContentAddition.Summary> contentAddition,
808 final RtpContentMap incomingContentAdd) {
809 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
810 final RtpContentMap modifiedContentMap =
811 getRemoteContentMap().addContent(incomingContentAdd, setup);
812 this.setRemoteContentMap(modifiedContentMap);
813
814 final SessionDescription offer;
815 try {
816 offer = SessionDescription.of(modifiedContentMap, isResponder());
817 } catch (final IllegalArgumentException | NullPointerException e) {
818 Log.d(
819 Config.LOGTAG,
820 id.getAccount().getJid().asBareJid()
821 + ": unable convert offer from content-add to SDP",
822 e);
823 webRTCWrapper.close();
824 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
825 return;
826 }
827 this.incomingContentAdd = null;
828 acceptContentAdd(contentAddition, offer);
829 }
830
831 private void acceptContentAdd(
832 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
833 final org.webrtc.SessionDescription sdp =
834 new org.webrtc.SessionDescription(
835 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
836 try {
837 this.webRTCWrapper.setRemoteDescription(sdp).get();
838
839 // TODO add tracks for 'media' where contentAddition.senders matches
840
841 // TODO if senders.sending(isInitiator())
842
843 this.webRTCWrapper.addTrack(Media.VIDEO);
844
845 // TODO add additional transceivers for recv only cases
846
847 final SessionDescription answer = setLocalSessionDescription();
848 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
849
850 final RtpContentMap contentAcceptMap =
851 rtpContentMap.toContentModification(
852 Collections2.transform(contentAddition, ca -> ca.name));
853
854 Log.d(
855 Config.LOGTAG,
856 id.getAccount().getJid().asBareJid()
857 + ": sending content-accept "
858 + ContentAddition.summary(contentAcceptMap));
859
860 addIceCandidatesFromBlackLog();
861
862 modifyLocalContentMap(rtpContentMap);
863 final ListenableFuture<RtpContentMap> future =
864 prepareOutgoingContentMap(contentAcceptMap);
865 Futures.addCallback(
866 future,
867 new FutureCallback<>() {
868 @Override
869 public void onSuccess(final RtpContentMap rtpContentMap) {
870 sendContentAccept(rtpContentMap);
871 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
872 }
873
874 @Override
875 public void onFailure(@NonNull final Throwable throwable) {
876 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
877 }
878 },
879 MoreExecutors.directExecutor());
880 } catch (final Exception e) {
881 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
882 webRTCWrapper.close();
883 sendSessionTerminate(Reason.FAILED_APPLICATION);
884 }
885 }
886
887 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
888 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
889 send(iq);
890 }
891
892 public synchronized void rejectContentAdd() {
893 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
894 if (incomingContentAdd == null) {
895 throw new IllegalStateException("No incoming content add");
896 }
897 this.incomingContentAdd = null;
898 updateEndUserState();
899 rejectContentAdd(incomingContentAdd);
900 }
901
902 private void rejectContentAdd(final RtpContentMap contentMap) {
903 final Iq iq =
904 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
905 Log.d(
906 Config.LOGTAG,
907 id.getAccount().getJid().asBareJid()
908 + ": rejecting content "
909 + ContentAddition.summary(contentMap));
910 send(iq);
911 }
912
913 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
914 final RtpContentMap existing = getRemoteContentMap();
915 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
916 final IceUdpTransportInfo.Credentials newCredentials;
917 try {
918 existingCredentials = existing.getCredentials();
919 newCredentials = rtpContentMap.getDistinctCredentials();
920 } catch (final IllegalStateException e) {
921 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
922 return false;
923 }
924 if (existingCredentials.contains(newCredentials)) {
925 return false;
926 }
927 // TODO an alternative approach is to check if we already got an iq result to our
928 // ICE-restart
929 // and if that's the case we are seeing an answer.
930 // This might be more spec compliant but also more error prone potentially
931 final boolean isSignalStateStable =
932 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
933 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
934 final boolean isOffer = rtpContentMap.emptyCandidates();
935 final RtpContentMap restartContentMap;
936 try {
937 if (isOffer) {
938 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
939 restartContentMap =
940 existing.modifiedCredentials(
941 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
942 } else {
943 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
944 Log.d(
945 Config.LOGTAG,
946 "received confirmation of ICE restart"
947 + newCredentials
948 + " peer_setup="
949 + setup);
950 // DTLS setup attribute needs to be rewritten to reflect current peer state
951 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
952 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
953 }
954 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
955 return isOffer;
956 } else {
957 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
958 respondWithTieBreak(jinglePacket);
959 return true;
960 }
961 } catch (final Exception exception) {
962 respondOk(jinglePacket);
963 final Throwable rootCause = Throwables.getRootCause(exception);
964 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
965 // If this happens a termination is already in progress
966 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
967 return true;
968 }
969 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
970 webRTCWrapper.close();
971 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
972 return true;
973 }
974 }
975
976 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
977 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
978 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
979 throw new IllegalStateException("Invalid peer setup");
980 }
981 return peerSetup;
982 }
983
984 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
985 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
986 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
987 }
988 this.peerDtlsSetup = setup;
989 }
990
991 private boolean applyIceRestart(
992 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
993 throws ExecutionException, InterruptedException {
994 final SessionDescription sessionDescription =
995 SessionDescription.of(restartContentMap, isResponder());
996 final org.webrtc.SessionDescription.Type type =
997 isOffer
998 ? org.webrtc.SessionDescription.Type.OFFER
999 : org.webrtc.SessionDescription.Type.ANSWER;
1000 org.webrtc.SessionDescription sdp =
1001 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1002 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1003 if (isInitiator()) {
1004 // We ignore the offer and respond with tie-break. This will clause the responder
1005 // not to apply the content map
1006 return false;
1007 }
1008 }
1009 webRTCWrapper.setRemoteDescription(sdp).get();
1010 setRemoteContentMap(restartContentMap);
1011 if (isOffer) {
1012 final SessionDescription localSessionDescription = setLocalSessionDescription();
1013 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1014 // We need to respond OK before sending any candidates
1015 respondOk(jinglePacket);
1016 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1017 } else {
1018 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1019 }
1020 return true;
1021 }
1022
1023 private void processCandidates(
1024 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1025 contents) {
1026 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1027 content : contents) {
1028 processCandidate(content);
1029 }
1030 }
1031
1032 private void processCandidate(
1033 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1034 content) {
1035 final RtpContentMap rtpContentMap = getRemoteContentMap();
1036 final List<String> indices = toIdentificationTags(rtpContentMap);
1037 final String sdpMid = content.getKey(); // aka content name
1038 final IceUdpTransportInfo transport = content.getValue().transport;
1039 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1040
1041 // TODO check that credentials remained the same
1042
1043 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1044 final String sdp;
1045 try {
1046 sdp = candidate.toSdpAttribute(credentials.ufrag);
1047 } catch (final IllegalArgumentException e) {
1048 Log.d(
1049 Config.LOGTAG,
1050 id.account.getJid().asBareJid()
1051 + ": ignoring invalid ICE candidate "
1052 + e.getMessage());
1053 continue;
1054 }
1055 final int mLineIndex = indices.indexOf(sdpMid);
1056 if (mLineIndex < 0) {
1057 Log.w(
1058 Config.LOGTAG,
1059 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1060 }
1061 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1062 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1063 this.webRTCWrapper.addIceCandidate(iceCandidate);
1064 }
1065 }
1066
1067 private RtpContentMap getRemoteContentMap() {
1068 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1069 }
1070
1071 private RtpContentMap getLocalContentMap() {
1072 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1073 }
1074
1075 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1076 final Group originalGroup = rtpContentMap.group;
1077 final List<String> identificationTags =
1078 originalGroup == null
1079 ? rtpContentMap.getNames()
1080 : originalGroup.getIdentificationTags();
1081 if (identificationTags.size() == 0) {
1082 Log.w(
1083 Config.LOGTAG,
1084 id.account.getJid().asBareJid()
1085 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1086 }
1087 return identificationTags;
1088 }
1089
1090 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1091 final Jingle jinglePacket, final boolean expectVerification) {
1092 try {
1093 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1094 } catch (final Exception e) {
1095 return Futures.immediateFailedFuture(e);
1096 }
1097 }
1098
1099 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1100 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1101 Log.d(
1102 Config.LOGTAG,
1103 "receiveRtpContentMap("
1104 + receivedContentMap.getClass().getSimpleName()
1105 + ",expectVerification="
1106 + expectVerification
1107 + ")");
1108 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1109 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1110 id.account
1111 .getAxolotlService()
1112 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1113 return Futures.transform(
1114 future,
1115 omemoVerifiedPayload -> {
1116 // TODO test if an exception here triggers a correct abort
1117 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1118 Log.d(
1119 Config.LOGTAG,
1120 id.account.getJid().asBareJid()
1121 + ": received verifiable DTLS fingerprint via "
1122 + omemoVerification);
1123 return omemoVerifiedPayload.getPayload();
1124 },
1125 MoreExecutors.directExecutor());
1126 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1127 return Futures.immediateFailedFuture(
1128 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1129 } else {
1130 return Futures.immediateFuture(receivedContentMap);
1131 }
1132 }
1133
1134 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1135 if (isInitiator()) {
1136 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1137 return;
1138 }
1139 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1140 Futures.addCallback(
1141 future,
1142 new FutureCallback<>() {
1143 @Override
1144 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1145 receiveSessionInitiate(jinglePacket, rtpContentMap);
1146 }
1147
1148 @Override
1149 public void onFailure(@NonNull final Throwable throwable) {
1150 respondOk(jinglePacket);
1151 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1152 }
1153 },
1154 MoreExecutors.directExecutor());
1155 }
1156
1157 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1158 try {
1159 contentMap.requireContentDescriptions();
1160 contentMap.requireDTLSFingerprint(true);
1161 } catch (final RuntimeException e) {
1162 Log.d(
1163 Config.LOGTAG,
1164 id.account.getJid().asBareJid() + ": improperly formatted contents",
1165 Throwables.getRootCause(e));
1166 respondOk(jinglePacket);
1167 sendSessionTerminate(Reason.of(e), e.getMessage());
1168 return;
1169 }
1170 Log.d(
1171 Config.LOGTAG,
1172 "processing session-init with " + contentMap.contents.size() + " contents");
1173 final State target;
1174 if (this.state == State.PROCEED) {
1175 Preconditions.checkState(
1176 proposedMedia != null && proposedMedia.size() > 0,
1177 "proposed media must be set when processing pre-approved session-initiate");
1178 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1179 sendSessionTerminate(
1180 Reason.SECURITY_ERROR,
1181 String.format(
1182 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1183 this.proposedMedia, contentMap.getMedia()));
1184 return;
1185 }
1186 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1187 } else {
1188 target = State.SESSION_INITIALIZED;
1189 setProposedMedia(contentMap.getMedia());
1190 }
1191 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1192 respondOk(jinglePacket);
1193 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1194 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1195 Log.d(
1196 Config.LOGTAG,
1197 id.account.getJid().asBareJid()
1198 + ": automatically accepting session-initiate");
1199 sendSessionAccept();
1200 } else {
1201 Log.d(
1202 Config.LOGTAG,
1203 id.account.getJid().asBareJid()
1204 + ": received not pre-approved session-initiate. start ringing");
1205 startRinging();
1206 }
1207 } else {
1208 Log.d(
1209 Config.LOGTAG,
1210 String.format(
1211 "%s: received session-initiate while in state %s",
1212 id.account.getJid().asBareJid(), state));
1213 terminateWithOutOfOrder(jinglePacket);
1214 }
1215 }
1216
1217 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1218 if (isResponder()) {
1219 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1220 return;
1221 }
1222 final ListenableFuture<RtpContentMap> future =
1223 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1224 Futures.addCallback(
1225 future,
1226 new FutureCallback<>() {
1227 @Override
1228 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1229 receiveSessionAccept(jinglePacket, rtpContentMap);
1230 }
1231
1232 @Override
1233 public void onFailure(@NonNull final Throwable throwable) {
1234 respondOk(jinglePacket);
1235 Log.d(
1236 Config.LOGTAG,
1237 id.account.getJid().asBareJid()
1238 + ": improperly formatted contents in session-accept",
1239 throwable);
1240 webRTCWrapper.close();
1241 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1242 }
1243 },
1244 MoreExecutors.directExecutor());
1245 }
1246
1247 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1248 try {
1249 contentMap.requireContentDescriptions();
1250 contentMap.requireDTLSFingerprint();
1251 } catch (final RuntimeException e) {
1252 respondOk(jinglePacket);
1253 Log.d(
1254 Config.LOGTAG,
1255 id.account.getJid().asBareJid()
1256 + ": improperly formatted contents in session-accept",
1257 e);
1258 webRTCWrapper.close();
1259 sendSessionTerminate(Reason.of(e), e.getMessage());
1260 return;
1261 }
1262 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1263 if (!initiatorMedia.equals(contentMap.getMedia())) {
1264 sendSessionTerminate(
1265 Reason.SECURITY_ERROR,
1266 String.format(
1267 "Your session-included included media %s but our session-initiate was %s",
1268 this.proposedMedia, contentMap.getMedia()));
1269 return;
1270 }
1271 Log.d(
1272 Config.LOGTAG,
1273 "processing session-accept with " + contentMap.contents.size() + " contents");
1274 if (transition(State.SESSION_ACCEPTED)) {
1275 respondOk(jinglePacket);
1276 receiveSessionAccept(contentMap);
1277 } else {
1278 Log.d(
1279 Config.LOGTAG,
1280 String.format(
1281 "%s: received session-accept while in state %s",
1282 id.account.getJid().asBareJid(), state));
1283 respondOk(jinglePacket);
1284 }
1285 }
1286
1287 private void receiveSessionAccept(final RtpContentMap contentMap) {
1288 this.responderRtpContentMap = contentMap;
1289 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1290 final SessionDescription sessionDescription;
1291 try {
1292 sessionDescription = SessionDescription.of(contentMap, false);
1293 } catch (final IllegalArgumentException | NullPointerException e) {
1294 Log.d(
1295 Config.LOGTAG,
1296 id.account.getJid().asBareJid()
1297 + ": unable convert offer from session-accept to SDP",
1298 e);
1299 webRTCWrapper.close();
1300 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1301 return;
1302 }
1303 final org.webrtc.SessionDescription answer =
1304 new org.webrtc.SessionDescription(
1305 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1306 try {
1307 this.webRTCWrapper.setRemoteDescription(answer).get();
1308 } catch (final Exception e) {
1309 Log.d(
1310 Config.LOGTAG,
1311 id.account.getJid().asBareJid()
1312 + ": unable to set remote description after receiving session-accept",
1313 Throwables.getRootCause(e));
1314 webRTCWrapper.close();
1315 sendSessionTerminate(
1316 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1317 return;
1318 }
1319 processCandidates(contentMap.contents.entrySet());
1320 }
1321
1322 private void sendSessionAccept() {
1323 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1324 if (rtpContentMap == null) {
1325 throw new IllegalStateException("initiator RTP Content Map has not been set");
1326 }
1327 final SessionDescription offer;
1328 try {
1329 offer = SessionDescription.of(rtpContentMap, true);
1330 } catch (final IllegalArgumentException | NullPointerException e) {
1331 Log.d(
1332 Config.LOGTAG,
1333 id.account.getJid().asBareJid()
1334 + ": unable convert offer from session-initiate to SDP",
1335 e);
1336 webRTCWrapper.close();
1337 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1338 return;
1339 }
1340 sendSessionAccept(rtpContentMap.getMedia(), offer);
1341 }
1342
1343 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1344 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1345 }
1346
1347 private synchronized void sendSessionAccept(
1348 final Set<Media> media,
1349 final SessionDescription offer,
1350 final List<PeerConnection.IceServer> iceServers) {
1351 if (isTerminated()) {
1352 Log.w(
1353 Config.LOGTAG,
1354 id.account.getJid().asBareJid()
1355 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1356 return;
1357 }
1358 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1359 try {
1360 setupWebRTC(media, iceServers, !includeCandidates);
1361 } catch (final WebRTCWrapper.InitializationException e) {
1362 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1363 webRTCWrapper.close();
1364 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1365 return;
1366 }
1367 final org.webrtc.SessionDescription sdp =
1368 new org.webrtc.SessionDescription(
1369 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1370 try {
1371 this.webRTCWrapper.setRemoteDescription(sdp).get();
1372 addIceCandidatesFromBlackLog();
1373 org.webrtc.SessionDescription webRTCSessionDescription =
1374 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1375 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1376 } catch (final Exception e) {
1377 failureToAcceptSession(e);
1378 }
1379 }
1380
1381 private void failureToAcceptSession(final Throwable throwable) {
1382 if (isTerminated()) {
1383 return;
1384 }
1385 final Throwable rootCause = Throwables.getRootCause(throwable);
1386 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1387 webRTCWrapper.close();
1388 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1389 }
1390
1391 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1392 if (isTerminated()) {
1393 return;
1394 }
1395 final Throwable rootCause = Throwables.getRootCause(throwable);
1396 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1397 webRTCWrapper.close();
1398 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1399 }
1400
1401 private void addIceCandidatesFromBlackLog() {
1402 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1403 while ((foo = this.pendingIceCandidates.poll()) != null) {
1404 processCandidate(foo);
1405 Log.d(
1406 Config.LOGTAG,
1407 id.account.getJid().asBareJid() + ": added candidate from back log");
1408 }
1409 }
1410
1411 private void prepareSessionAccept(
1412 final org.webrtc.SessionDescription webRTCSessionDescription,
1413 final boolean includeCandidates) {
1414 final SessionDescription sessionDescription =
1415 SessionDescription.parse(webRTCSessionDescription.description);
1416 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1417 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1418 if (includeCandidates) {
1419 candidates = parseCandidates(sessionDescription);
1420 } else {
1421 candidates = ImmutableMultimap.of();
1422 }
1423 this.responderRtpContentMap = respondingRtpContentMap;
1424 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1425 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1426 prepareOutgoingContentMap(respondingRtpContentMap);
1427 Futures.addCallback(
1428 outgoingContentMapFuture,
1429 new FutureCallback<>() {
1430 @Override
1431 public void onSuccess(final RtpContentMap outgoingContentMap) {
1432 if (includeCandidates) {
1433 Log.d(
1434 Config.LOGTAG,
1435 "including "
1436 + candidates.size()
1437 + " candidates in session accept");
1438 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1439 } else {
1440 sendSessionAccept(outgoingContentMap);
1441 }
1442 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1443 }
1444
1445 @Override
1446 public void onFailure(@NonNull Throwable throwable) {
1447 failureToAcceptSession(throwable);
1448 }
1449 },
1450 MoreExecutors.directExecutor());
1451 }
1452
1453 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1454 if (isTerminated()) {
1455 Log.w(
1456 Config.LOGTAG,
1457 id.account.getJid().asBareJid()
1458 + ": preparing session accept was too slow. already terminated. nothing to do.");
1459 return;
1460 }
1461 transitionOrThrow(State.SESSION_ACCEPTED);
1462 final Iq sessionAccept =
1463 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1464 send(sessionAccept);
1465 }
1466
1467 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1468 final RtpContentMap rtpContentMap) {
1469 if (this.omemoVerification.hasDeviceId()) {
1470 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1471 verifiedPayloadFuture =
1472 id.account
1473 .getAxolotlService()
1474 .encrypt(
1475 rtpContentMap,
1476 id.with,
1477 omemoVerification.getDeviceId());
1478 return Futures.transform(
1479 verifiedPayloadFuture,
1480 verifiedPayload -> {
1481 omemoVerification.setOrEnsureEqual(verifiedPayload);
1482 return verifiedPayload.getPayload();
1483 },
1484 MoreExecutors.directExecutor());
1485 } else {
1486 return Futures.immediateFuture(rtpContentMap);
1487 }
1488 }
1489
1490 synchronized void deliveryMessage(
1491 final Jid from,
1492 final Element message,
1493 final String serverMessageId,
1494 final long timestamp) {
1495 Log.d(
1496 Config.LOGTAG,
1497 id.account.getJid().asBareJid()
1498 + ": delivered message to JingleRtpConnection "
1499 + message);
1500 switch (message.getName()) {
1501 case "propose" -> receivePropose(
1502 from, Propose.upgrade(message), serverMessageId, timestamp);
1503 case "proceed" -> receiveProceed(
1504 from, Proceed.upgrade(message), serverMessageId, timestamp);
1505 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1506 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1507 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1508 }
1509 }
1510
1511 void deliverFailedProceed(final String message) {
1512 Log.d(
1513 Config.LOGTAG,
1514 id.account.getJid().asBareJid()
1515 + ": receive message error for proceed message ("
1516 + Strings.nullToEmpty(message)
1517 + ")");
1518 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1519 webRTCWrapper.close();
1520 Log.d(
1521 Config.LOGTAG,
1522 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1523 this.finish();
1524 }
1525 }
1526
1527 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1528 final boolean originatedFromMyself =
1529 from.asBareJid().equals(id.account.getJid().asBareJid());
1530 if (originatedFromMyself) {
1531 if (transition(State.ACCEPTED)) {
1532 acceptedOnOtherDevice(serverMsgId, timestamp);
1533 } else {
1534 Log.d(
1535 Config.LOGTAG,
1536 id.account.getJid().asBareJid()
1537 + ": unable to transition to accept because already in state="
1538 + this.state);
1539 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1540 }
1541 } else {
1542 Log.d(
1543 Config.LOGTAG,
1544 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1545 }
1546 }
1547
1548 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1549 if (serverMsgId != null) {
1550 this.message.setServerMsgId(serverMsgId);
1551 }
1552 this.message.setTime(timestamp);
1553 this.message.setCarbon(true); // indicate that call was accepted on other device
1554 this.writeLogMessageSuccess(0);
1555 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1556 this.finish();
1557 }
1558
1559 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1560 final boolean originatedFromMyself =
1561 from.asBareJid().equals(id.account.getJid().asBareJid());
1562 // reject from another one of my clients
1563 if (originatedFromMyself) {
1564 receiveRejectFromMyself(serverMsgId, timestamp);
1565 } else if (isInitiator()) {
1566 if (from.equals(id.with)) {
1567 receiveRejectFromResponder();
1568 } else {
1569 Log.d(
1570 Config.LOGTAG,
1571 id.account.getJid()
1572 + ": ignoring reject from "
1573 + from
1574 + " for session with "
1575 + id.with);
1576 }
1577 } else {
1578 Log.d(
1579 Config.LOGTAG,
1580 id.account.getJid()
1581 + ": ignoring reject from "
1582 + from
1583 + " for session with "
1584 + id.with);
1585 }
1586 }
1587
1588 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1589 if (transition(State.REJECTED)) {
1590 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1591 this.finish();
1592 if (serverMsgId != null) {
1593 this.message.setServerMsgId(serverMsgId);
1594 }
1595 this.message.setTime(timestamp);
1596 this.message.setCarbon(true); // indicate that call was rejected on other device
1597 writeLogMessageMissed();
1598 } else {
1599 Log.d(
1600 Config.LOGTAG,
1601 "not able to transition into REJECTED because already in " + this.state);
1602 }
1603 }
1604
1605 private void receiveRejectFromResponder() {
1606 if (isInState(State.PROCEED)) {
1607 Log.d(
1608 Config.LOGTAG,
1609 id.account.getJid()
1610 + ": received reject while still in proceed. callee reconsidered");
1611 closeTransitionLogFinish(State.REJECTED_RACED);
1612 return;
1613 }
1614 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1615 Log.d(
1616 Config.LOGTAG,
1617 id.account.getJid()
1618 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1619 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1620 return;
1621 }
1622 Log.d(
1623 Config.LOGTAG,
1624 id.account.getJid()
1625 + ": ignoring reject from responder because already in state "
1626 + this.state);
1627 }
1628
1629 private void receivePropose(
1630 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1631 final boolean originatedFromMyself =
1632 from.asBareJid().equals(id.account.getJid().asBareJid());
1633 if (originatedFromMyself) {
1634 Log.d(
1635 Config.LOGTAG,
1636 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1637 } else if (transition(
1638 State.PROPOSED,
1639 () -> {
1640 final Collection<RtpDescription> descriptions =
1641 Collections2.transform(
1642 Collections2.filter(
1643 propose.getDescriptions(),
1644 d -> d instanceof RtpDescription),
1645 input -> (RtpDescription) input);
1646 final Collection<Media> media =
1647 Collections2.transform(descriptions, RtpDescription::getMedia);
1648 Preconditions.checkState(
1649 !media.contains(Media.UNKNOWN),
1650 "RTP descriptions contain unknown media");
1651 Log.d(
1652 Config.LOGTAG,
1653 id.account.getJid().asBareJid()
1654 + ": received session proposal from "
1655 + from
1656 + " for "
1657 + media);
1658 this.setProposedMedia(Sets.newHashSet(media));
1659 })) {
1660 if (serverMsgId != null) {
1661 this.message.setServerMsgId(serverMsgId);
1662 }
1663 this.message.setTime(timestamp);
1664 startRinging();
1665 // in environments where we always use discovery timeouts we always want to respond with
1666 // 'ringing'
1667 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1668 || id.getContact().showInContactList()) {
1669 sendJingleMessage("ringing");
1670 }
1671 } else {
1672 Log.d(
1673 Config.LOGTAG,
1674 id.account.getJid()
1675 + ": ignoring session proposal because already in "
1676 + state);
1677 }
1678 }
1679
1680 private void startRinging() {
1681 this.callIntegration.setRinging();
1682 Log.d(
1683 Config.LOGTAG,
1684 id.account.getJid().asBareJid()
1685 + ": received call from "
1686 + id.with
1687 + ". start ringing");
1688 ringingTimeoutFuture =
1689 jingleConnectionManager.schedule(
1690 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1691 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1692 return;
1693 }
1694 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1695 }
1696
1697 private synchronized void ringingTimeout() {
1698 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1699 switch (this.state) {
1700 case PROPOSED -> {
1701 message.markUnread();
1702 rejectCallFromProposed();
1703 }
1704 case SESSION_INITIALIZED -> {
1705 message.markUnread();
1706 rejectCallFromSessionInitiate();
1707 }
1708 }
1709 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1710 }
1711
1712 private void cancelRingingTimeout() {
1713 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1714 if (future != null && !future.isCancelled()) {
1715 future.cancel(false);
1716 }
1717 }
1718
1719 private void receiveProceed(
1720 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1721 final Set<Media> media =
1722 Preconditions.checkNotNull(
1723 this.proposedMedia, "Proposed media has to be set before handling proceed");
1724 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1725 if (from.equals(id.with)) {
1726 if (isInitiator()) {
1727 if (transition(State.PROCEED)) {
1728 if (serverMsgId != null) {
1729 this.message.setServerMsgId(serverMsgId);
1730 }
1731 this.message.setTime(timestamp);
1732 final Integer remoteDeviceId = proceed.getDeviceId();
1733 if (isOmemoEnabled()) {
1734 this.omemoVerification.setDeviceId(remoteDeviceId);
1735 } else {
1736 if (remoteDeviceId != null) {
1737 Log.d(
1738 Config.LOGTAG,
1739 id.account.getJid().asBareJid()
1740 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1741 }
1742 this.omemoVerification.setDeviceId(null);
1743 }
1744 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1745 } else {
1746 Log.d(
1747 Config.LOGTAG,
1748 String.format(
1749 "%s: ignoring proceed because already in %s",
1750 id.account.getJid().asBareJid(), this.state));
1751 }
1752 } else {
1753 Log.d(
1754 Config.LOGTAG,
1755 String.format(
1756 "%s: ignoring proceed because we were not initializing",
1757 id.account.getJid().asBareJid()));
1758 }
1759 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1760 if (transition(State.ACCEPTED)) {
1761 Log.d(
1762 Config.LOGTAG,
1763 id.account.getJid().asBareJid()
1764 + ": moved session with "
1765 + id.with
1766 + " into state accepted after received carbon copied proceed");
1767 acceptedOnOtherDevice(serverMsgId, timestamp);
1768 }
1769 } else {
1770 Log.d(
1771 Config.LOGTAG,
1772 String.format(
1773 "%s: ignoring proceed from %s. was expected from %s",
1774 id.account.getJid().asBareJid(), from, id.with));
1775 }
1776 }
1777
1778 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1779 if (from.equals(id.with)) {
1780 final State target =
1781 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1782 if (transition(target)) {
1783 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1784 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1785 Log.d(
1786 Config.LOGTAG,
1787 id.account.getJid().asBareJid()
1788 + ": session with "
1789 + id.with
1790 + " has been retracted (serverMsgId="
1791 + serverMsgId
1792 + ")");
1793 if (serverMsgId != null) {
1794 this.message.setServerMsgId(serverMsgId);
1795 }
1796 this.message.setTime(timestamp);
1797 if (target == State.RETRACTED) {
1798 this.message.markUnread();
1799 }
1800 writeLogMessageMissed();
1801 finish();
1802 } else {
1803 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1804 }
1805 } else {
1806 // TODO parse retract from self
1807 Log.d(
1808 Config.LOGTAG,
1809 id.account.getJid().asBareJid()
1810 + ": received retract from "
1811 + from
1812 + ". expected retract from"
1813 + id.with
1814 + ". ignoring");
1815 }
1816 }
1817
1818 public void sendSessionInitiate() {
1819 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1820 }
1821
1822 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1823 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1824 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1825 }
1826
1827 private synchronized void sendSessionInitiate(
1828 final Set<Media> media,
1829 final State targetState,
1830 final List<PeerConnection.IceServer> iceServers) {
1831 if (isTerminated()) {
1832 Log.w(
1833 Config.LOGTAG,
1834 id.account.getJid().asBareJid()
1835 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1836 return;
1837 }
1838 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1839 try {
1840 setupWebRTC(media, iceServers, !includeCandidates);
1841 } catch (final WebRTCWrapper.InitializationException e) {
1842 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1843 webRTCWrapper.close();
1844 sendRetract(Reason.ofThrowable(e));
1845 return;
1846 }
1847 try {
1848 org.webrtc.SessionDescription webRTCSessionDescription =
1849 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1850 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1851 } catch (final Exception e) {
1852 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1853 // exceptions
1854 failureToInitiateSession(e, targetState);
1855 }
1856 }
1857
1858 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1859 if (isTerminated()) {
1860 return;
1861 }
1862 Log.d(
1863 Config.LOGTAG,
1864 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1865 Throwables.getRootCause(throwable));
1866 webRTCWrapper.close();
1867 final Reason reason = Reason.ofThrowable(throwable);
1868 if (isInState(targetState)) {
1869 sendSessionTerminate(reason, throwable.getMessage());
1870 } else {
1871 sendRetract(reason);
1872 }
1873 }
1874
1875 private void sendRetract(final Reason reason) {
1876 // TODO embed reason into retract
1877 sendJingleMessage("retract", id.with.asBareJid());
1878 transitionOrThrow(reasonToState(reason));
1879 this.finish();
1880 }
1881
1882 private void prepareSessionInitiate(
1883 final org.webrtc.SessionDescription webRTCSessionDescription,
1884 final boolean includeCandidates,
1885 final State targetState) {
1886 final SessionDescription sessionDescription =
1887 SessionDescription.parse(webRTCSessionDescription.description);
1888 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1889 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1890 if (includeCandidates) {
1891 candidates = parseCandidates(sessionDescription);
1892 } else {
1893 candidates = ImmutableMultimap.of();
1894 }
1895 this.initiatorRtpContentMap = rtpContentMap;
1896 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1897 encryptSessionInitiate(rtpContentMap);
1898 Futures.addCallback(
1899 outgoingContentMapFuture,
1900 new FutureCallback<>() {
1901 @Override
1902 public void onSuccess(final RtpContentMap outgoingContentMap) {
1903 if (includeCandidates) {
1904 Log.d(
1905 Config.LOGTAG,
1906 "including "
1907 + candidates.size()
1908 + " candidates in session initiate");
1909 sendSessionInitiate(
1910 outgoingContentMap.withCandidates(candidates), targetState);
1911 } else {
1912 sendSessionInitiate(outgoingContentMap, targetState);
1913 }
1914 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1915 }
1916
1917 @Override
1918 public void onFailure(@NonNull final Throwable throwable) {
1919 failureToInitiateSession(throwable, targetState);
1920 }
1921 },
1922 MoreExecutors.directExecutor());
1923 }
1924
1925 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1926 if (isTerminated()) {
1927 Log.w(
1928 Config.LOGTAG,
1929 id.account.getJid().asBareJid()
1930 + ": preparing session was too slow. already terminated. nothing to do.");
1931 return;
1932 }
1933 this.transitionOrThrow(targetState);
1934 final Iq sessionInitiate =
1935 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1936 send(sessionInitiate);
1937 }
1938
1939 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1940 final RtpContentMap rtpContentMap) {
1941 if (this.omemoVerification.hasDeviceId()) {
1942 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1943 verifiedPayloadFuture =
1944 id.account
1945 .getAxolotlService()
1946 .encrypt(
1947 rtpContentMap,
1948 id.with,
1949 omemoVerification.getDeviceId());
1950 final ListenableFuture<RtpContentMap> future =
1951 Futures.transform(
1952 verifiedPayloadFuture,
1953 verifiedPayload -> {
1954 omemoVerification.setSessionFingerprint(
1955 verifiedPayload.getFingerprint());
1956 return verifiedPayload.getPayload();
1957 },
1958 MoreExecutors.directExecutor());
1959 if (Config.REQUIRE_RTP_VERIFICATION) {
1960 return future;
1961 }
1962 return Futures.catching(
1963 future,
1964 CryptoFailedException.class,
1965 e -> {
1966 Log.w(
1967 Config.LOGTAG,
1968 id.account.getJid().asBareJid()
1969 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1970 e);
1971 return rtpContentMap;
1972 },
1973 MoreExecutors.directExecutor());
1974 } else {
1975 return Futures.immediateFuture(rtpContentMap);
1976 }
1977 }
1978
1979 protected void sendSessionTerminate(final Reason reason) {
1980 sendSessionTerminate(reason, null);
1981 }
1982
1983 protected void sendSessionTerminate(final Reason reason, final String text) {
1984 sendSessionTerminate(reason, text, this::writeLogMessage);
1985 sendJingleMessageFinish(reason);
1986 }
1987
1988 private void sendTransportInfo(
1989 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1990 final RtpContentMap transportInfo;
1991 try {
1992 final RtpContentMap rtpContentMap =
1993 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1994 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1995 } catch (final Exception e) {
1996 Log.d(
1997 Config.LOGTAG,
1998 id.account.getJid().asBareJid()
1999 + ": unable to prepare transport-info from candidate for content="
2000 + contentName);
2001 return;
2002 }
2003 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2004 send(iq);
2005 }
2006
2007 public RtpEndUserState getEndUserState() {
2008 switch (this.state) {
2009 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2010 if (isInitiator()) {
2011 return RtpEndUserState.RINGING;
2012 } else {
2013 return RtpEndUserState.INCOMING_CALL;
2014 }
2015 }
2016 case PROCEED -> {
2017 if (isInitiator()) {
2018 return RtpEndUserState.RINGING;
2019 } else {
2020 return RtpEndUserState.ACCEPTING_CALL;
2021 }
2022 }
2023 case SESSION_INITIALIZED_PRE_APPROVED -> {
2024 if (isInitiator()) {
2025 return RtpEndUserState.RINGING;
2026 } else {
2027 return RtpEndUserState.CONNECTING;
2028 }
2029 }
2030 case SESSION_ACCEPTED -> {
2031 final ContentAddition ca = getPendingContentAddition();
2032 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2033 return RtpEndUserState.INCOMING_CONTENT_ADD;
2034 }
2035 return getPeerConnectionStateAsEndUserState();
2036 }
2037 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.DECLINED_OR_BUSY;
2040 } else {
2041 return RtpEndUserState.ENDED;
2042 }
2043 }
2044 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2045 return RtpEndUserState.ENDED;
2046 }
2047 case RETRACTED_RACED -> {
2048 if (isInitiator()) {
2049 return RtpEndUserState.ENDED;
2050 } else {
2051 return RtpEndUserState.RETRACTED;
2052 }
2053 }
2054 case TERMINATED_CONNECTIVITY_ERROR -> {
2055 return zeroDuration()
2056 ? RtpEndUserState.CONNECTIVITY_ERROR
2057 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2058 }
2059 case TERMINATED_APPLICATION_FAILURE -> {
2060 return RtpEndUserState.APPLICATION_ERROR;
2061 }
2062 case TERMINATED_SECURITY_ERROR -> {
2063 return RtpEndUserState.SECURITY_ERROR;
2064 }
2065 }
2066 throw new IllegalStateException(
2067 String.format("%s has no equivalent EndUserState", this.state));
2068 }
2069
2070 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2071 final PeerConnection.PeerConnectionState state;
2072 try {
2073 state = webRTCWrapper.getState();
2074 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2075 // We usually close the WebRTCWrapper *before* transitioning so we might still
2076 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2077 return RtpEndUserState.ENDING_CALL;
2078 }
2079 return switch (state) {
2080 case CONNECTED -> RtpEndUserState.CONNECTED;
2081 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2082 case CLOSED -> RtpEndUserState.ENDING_CALL;
2083 default -> zeroDuration()
2084 ? RtpEndUserState.CONNECTIVITY_ERROR
2085 : RtpEndUserState.RECONNECTING;
2086 };
2087 }
2088
2089 private boolean isPeerConnectionConnected() {
2090 try {
2091 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2092 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2093 return false;
2094 }
2095 }
2096
2097 private void updateCallIntegrationState() {
2098 switch (this.state) {
2099 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2100 if (isInitiator()) {
2101 this.callIntegration.setDialing();
2102 } else {
2103 this.callIntegration.setRinging();
2104 }
2105 }
2106 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2107 if (isInitiator()) {
2108 this.callIntegration.setDialing();
2109 } else {
2110 this.callIntegration.setInitialized();
2111 }
2112 }
2113 case SESSION_ACCEPTED -> {
2114 if (isPeerConnectionConnected()) {
2115 this.callIntegration.setActive();
2116 } else {
2117 this.callIntegration.setInitialized();
2118 }
2119 }
2120 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2121 if (isInitiator()) {
2122 this.callIntegration.busy();
2123 } else {
2124 this.callIntegration.rejected();
2125 }
2126 }
2127 case TERMINATED_SUCCESS -> this.callIntegration.success();
2128 case ACCEPTED -> this.callIntegration.accepted();
2129 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2130 .retracted();
2131 case TERMINATED_CONNECTIVITY_ERROR,
2132 TERMINATED_APPLICATION_FAILURE,
2133 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2134 default -> throw new IllegalStateException(
2135 String.format("%s is not handled", this.state));
2136 }
2137 }
2138
2139 public ContentAddition getPendingContentAddition() {
2140 final RtpContentMap in = this.incomingContentAdd;
2141 final RtpContentMap out = this.outgoingContentAdd;
2142 if (out != null) {
2143 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2144 } else if (in != null) {
2145 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2146 } else {
2147 return null;
2148 }
2149 }
2150
2151 public Set<Media> getMedia() {
2152 final State current = getState();
2153 if (current == State.NULL) {
2154 if (isInitiator()) {
2155 return Preconditions.checkNotNull(
2156 this.proposedMedia, "RTP connection has not been initialized properly");
2157 }
2158 throw new IllegalStateException("RTP connection has not been initialized yet");
2159 }
2160 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2161 return Preconditions.checkNotNull(
2162 this.proposedMedia, "RTP connection has not been initialized properly");
2163 }
2164 final RtpContentMap localContentMap = getLocalContentMap();
2165 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2166 if (localContentMap != null) {
2167 return localContentMap.getMedia();
2168 } else if (initiatorContentMap != null) {
2169 return initiatorContentMap.getMedia();
2170 } else if (isTerminated()) {
2171 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2172 } else {
2173 return Preconditions.checkNotNull(
2174 this.proposedMedia, "RTP connection has not been initialized properly");
2175 }
2176 }
2177
2178 public boolean isVerified() {
2179 final String fingerprint = this.omemoVerification.getFingerprint();
2180 if (fingerprint == null) {
2181 return false;
2182 }
2183 final FingerprintStatus status =
2184 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2185 return status != null && status.isVerified();
2186 }
2187
2188 public boolean addMedia(final Media media) {
2189 final Set<Media> currentMedia = getMedia();
2190 if (currentMedia.contains(media)) {
2191 throw new IllegalStateException(String.format("%s has already been proposed", media));
2192 }
2193 // TODO add state protection - can only add while ACCEPTED or so
2194 Log.d(Config.LOGTAG, "adding media: " + media);
2195 return webRTCWrapper.addTrack(media);
2196 }
2197
2198 public synchronized void acceptCall() {
2199 switch (this.state) {
2200 case PROPOSED -> {
2201 cancelRingingTimeout();
2202 acceptCallFromProposed();
2203 }
2204 case SESSION_INITIALIZED -> {
2205 cancelRingingTimeout();
2206 acceptCallFromSessionInitialized();
2207 }
2208 case ACCEPTED -> Log.w(
2209 Config.LOGTAG,
2210 id.account.getJid().asBareJid()
2211 + ": the call has already been accepted with another client. UI was just lagging behind");
2212 case PROCEED, SESSION_ACCEPTED -> Log.w(
2213 Config.LOGTAG,
2214 id.account.getJid().asBareJid()
2215 + ": the call has already been accepted. user probably double tapped the UI");
2216 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2217 }
2218 }
2219
2220 public synchronized void rejectCall() {
2221 if (isTerminated()) {
2222 Log.w(
2223 Config.LOGTAG,
2224 id.account.getJid().asBareJid()
2225 + ": received rejectCall() when session has already been terminated. nothing to do");
2226 return;
2227 }
2228 switch (this.state) {
2229 case PROPOSED -> rejectCallFromProposed();
2230 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2231 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2232 }
2233 }
2234
2235 public synchronized void integrationFailure() {
2236 final var state = getState();
2237 if (state == State.PROPOSED) {
2238 Log.e(
2239 Config.LOGTAG,
2240 id.account.getJid().asBareJid()
2241 + ": failed call integration in state proposed");
2242 rejectCallFromProposed();
2243 } else if (state == State.SESSION_INITIALIZED) {
2244 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2245 this.webRTCWrapper.close();
2246 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2247 } else {
2248 throw new IllegalStateException(
2249 String.format("Can not fail integration in state %s", state));
2250 }
2251 }
2252
2253 public synchronized void endCall() {
2254 if (isTerminated()) {
2255 Log.w(
2256 Config.LOGTAG,
2257 id.account.getJid().asBareJid()
2258 + ": received endCall() when session has already been terminated. nothing to do");
2259 return;
2260 }
2261 if (isInState(State.PROPOSED) && isResponder()) {
2262 rejectCallFromProposed();
2263 return;
2264 }
2265 if (isInState(State.PROCEED)) {
2266 if (isInitiator()) {
2267 retractFromProceed();
2268 } else {
2269 rejectCallFromProceed();
2270 }
2271 return;
2272 }
2273 if (isInitiator()
2274 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2275 this.webRTCWrapper.close();
2276 sendSessionTerminate(Reason.CANCEL);
2277 return;
2278 }
2279 if (isInState(State.SESSION_INITIALIZED)) {
2280 rejectCallFromSessionInitiate();
2281 return;
2282 }
2283 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2284 this.webRTCWrapper.close();
2285 sendSessionTerminate(Reason.SUCCESS);
2286 return;
2287 }
2288 if (isInState(
2289 State.TERMINATED_APPLICATION_FAILURE,
2290 State.TERMINATED_CONNECTIVITY_ERROR,
2291 State.TERMINATED_DECLINED_OR_BUSY)) {
2292 Log.d(
2293 Config.LOGTAG,
2294 "ignoring request to end call because already in state " + this.state);
2295 return;
2296 }
2297 throw new IllegalStateException(
2298 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2299 }
2300
2301 private void retractFromProceed() {
2302 Log.d(Config.LOGTAG, "retract from proceed");
2303 this.sendJingleMessage("retract");
2304 closeTransitionLogFinish(State.RETRACTED_RACED);
2305 }
2306
2307 private void closeTransitionLogFinish(final State state) {
2308 this.webRTCWrapper.close();
2309 transitionOrThrow(state);
2310 writeLogMessage(state);
2311 finish();
2312 }
2313
2314 private void setupWebRTC(
2315 final Set<Media> media,
2316 final List<PeerConnection.IceServer> iceServers,
2317 final boolean trickle)
2318 throws WebRTCWrapper.InitializationException {
2319 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2320 this.webRTCWrapper.setup(this.xmppConnectionService);
2321 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2322 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2323 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2324 }
2325
2326 private void acceptCallFromProposed() {
2327 transitionOrThrow(State.PROCEED);
2328 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2329 this.callIntegration.startAudioRouting();
2330 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2331 this.sendJingleMessage("proceed");
2332 }
2333
2334 private void rejectCallFromProposed() {
2335 transitionOrThrow(State.REJECTED);
2336 writeLogMessageMissed();
2337 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2338 this.sendJingleMessage("reject");
2339 finish();
2340 }
2341
2342 private void rejectCallFromProceed() {
2343 this.sendJingleMessage("reject");
2344 closeTransitionLogFinish(State.REJECTED_RACED);
2345 }
2346
2347 private void rejectCallFromSessionInitiate() {
2348 webRTCWrapper.close();
2349 sendSessionTerminate(Reason.DECLINE);
2350 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2351 }
2352
2353 private void sendJingleMessage(final String action) {
2354 sendJingleMessage(action, id.with);
2355 }
2356
2357 private void sendJingleMessage(final String action, final Jid to) {
2358 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2359 messagePacket.setType(
2360 im.conversations.android.xmpp.model.stanza.Message.Type
2361 .CHAT); // we want to carbon copy those
2362 messagePacket.setTo(to);
2363 final Element intent =
2364 messagePacket
2365 .addChild(action, Namespace.JINGLE_MESSAGE)
2366 .setAttribute("id", id.sessionId);
2367 if ("proceed".equals(action)) {
2368 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2369 if (isOmemoEnabled()) {
2370 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2371 final Element device =
2372 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2373 device.setAttribute("id", deviceId);
2374 }
2375 }
2376 messagePacket.addChild("store", "urn:xmpp:hints");
2377 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2378 }
2379
2380 private void sendJingleMessageFinish(final Reason reason) {
2381 final var account = id.getAccount();
2382 final var messagePacket =
2383 xmppConnectionService
2384 .getMessageGenerator()
2385 .sessionFinish(id.with, id.sessionId, reason);
2386 xmppConnectionService.sendMessagePacket(account, messagePacket);
2387 }
2388
2389 private boolean isOmemoEnabled() {
2390 final Conversational conversational = message.getConversation();
2391 if (conversational instanceof Conversation) {
2392 return ((Conversation) conversational).getNextEncryption()
2393 == Message.ENCRYPTION_AXOLOTL;
2394 }
2395 return false;
2396 }
2397
2398 private void acceptCallFromSessionInitialized() {
2399 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2400 this.callIntegration.startAudioRouting();
2401 sendSessionAccept();
2402 }
2403
2404 @Override
2405 protected synchronized boolean transition(final State target, final Runnable runnable) {
2406 if (super.transition(target, runnable)) {
2407 updateEndUserState();
2408 updateOngoingCallNotification();
2409 return true;
2410 } else {
2411 return false;
2412 }
2413 }
2414
2415 @Override
2416 public void onIceCandidate(final IceCandidate iceCandidate) {
2417 final RtpContentMap rtpContentMap =
2418 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2419 final IceUdpTransportInfo.Credentials credentials;
2420 try {
2421 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2422 } catch (final IllegalArgumentException e) {
2423 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2424 return;
2425 }
2426 final String uFrag = credentials.ufrag;
2427 final IceUdpTransportInfo.Candidate candidate =
2428 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2429 if (candidate == null) {
2430 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2431 return;
2432 }
2433 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2434 sendTransportInfo(iceCandidate.sdpMid, candidate);
2435 }
2436
2437 @Override
2438 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2439 Log.d(
2440 Config.LOGTAG,
2441 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2442 this.stateHistory.add(newState);
2443 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2444 this.sessionDuration.start();
2445 updateOngoingCallNotification();
2446 } else if (this.sessionDuration.isRunning()) {
2447 this.sessionDuration.stop();
2448 updateOngoingCallNotification();
2449 }
2450
2451 final boolean neverConnected =
2452 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2453
2454 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2455 if (neverConnected) {
2456 if (isTerminated()) {
2457 Log.d(
2458 Config.LOGTAG,
2459 id.account.getJid().asBareJid()
2460 + ": not sending session-terminate after connectivity error because session is already in state "
2461 + this.state);
2462 return;
2463 }
2464 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2465 return;
2466 } else {
2467 this.restartIce();
2468 }
2469 }
2470 updateEndUserState();
2471 }
2472
2473 private void restartIce() {
2474 this.stateHistory.clear();
2475 this.webRTCWrapper.restartIceAsync();
2476 }
2477
2478 @Override
2479 public void onRenegotiationNeeded() {
2480 this.webRTCWrapper.execute(this::renegotiate);
2481 }
2482
2483 private synchronized void renegotiate() {
2484 final SessionDescription sessionDescription;
2485 try {
2486 sessionDescription = setLocalSessionDescription();
2487 } catch (final Exception e) {
2488 final Throwable cause = Throwables.getRootCause(e);
2489 webRTCWrapper.close();
2490 if (isTerminated()) {
2491 Log.d(
2492 Config.LOGTAG,
2493 "failed to renegotiate. session was already terminated",
2494 cause);
2495 return;
2496 }
2497 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2498 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2499 return;
2500 }
2501 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2502 final RtpContentMap currentContentMap = getLocalContentMap();
2503 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2504 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2505
2506 Log.d(
2507 Config.LOGTAG,
2508 id.getAccount().getJid().asBareJid()
2509 + ": renegotiate. iceRestart="
2510 + iceRestart
2511 + " content id diff="
2512 + diff);
2513
2514 if (diff.hasModifications() && iceRestart) {
2515 webRTCWrapper.close();
2516 sendSessionTerminate(
2517 Reason.FAILED_APPLICATION,
2518 "WebRTC unexpectedly tried to modify content and transport at once");
2519 return;
2520 }
2521
2522 if (iceRestart) {
2523 initiateIceRestart(rtpContentMap);
2524 return;
2525 } else if (diff.isEmpty()) {
2526 Log.d(
2527 Config.LOGTAG,
2528 "renegotiation. nothing to do. SignalingState="
2529 + this.webRTCWrapper.getSignalingState());
2530 }
2531
2532 if (diff.added.isEmpty()) {
2533 return;
2534 }
2535 modifyLocalContentMap(rtpContentMap);
2536 sendContentAdd(rtpContentMap, diff.added);
2537 }
2538
2539 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2540 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2541 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2542 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2543 iq.setTo(id.with);
2544 xmppConnectionService.sendIqPacket(
2545 id.account,
2546 iq,
2547 (response) -> {
2548 if (response.getType() == Iq.Type.RESULT) {
2549 Log.d(Config.LOGTAG, "received success to our ice restart");
2550 setLocalContentMap(rtpContentMap);
2551 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2552 return;
2553 }
2554 if (response.getType() == Iq.Type.ERROR) {
2555 if (isTieBreak(response)) {
2556 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2557 return;
2558 }
2559 handleIqErrorResponse(response);
2560 }
2561 if (response.getType() == Iq.Type.TIMEOUT) {
2562 handleIqTimeoutResponse(response);
2563 }
2564 });
2565 }
2566
2567 private boolean isTieBreak(final Iq response) {
2568 final Element error = response.findChild("error");
2569 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2570 }
2571
2572 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2573 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2574 this.outgoingContentAdd = contentAdd;
2575 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2576 prepareOutgoingContentMap(contentAdd);
2577 Futures.addCallback(
2578 outgoingContentMapFuture,
2579 new FutureCallback<>() {
2580 @Override
2581 public void onSuccess(final RtpContentMap outgoingContentMap) {
2582 sendContentAdd(outgoingContentMap);
2583 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2584 }
2585
2586 @Override
2587 public void onFailure(@NonNull Throwable throwable) {
2588 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2589 }
2590 },
2591 MoreExecutors.directExecutor());
2592 }
2593
2594 private void sendContentAdd(final RtpContentMap contentAdd) {
2595
2596 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2597 iq.setTo(id.with);
2598 xmppConnectionService.sendIqPacket(
2599 id.account,
2600 iq,
2601 (response) -> {
2602 if (response.getType() == Iq.Type.RESULT) {
2603 Log.d(
2604 Config.LOGTAG,
2605 id.getAccount().getJid().asBareJid()
2606 + ": received ACK to our content-add");
2607 return;
2608 }
2609 if (response.getType() == Iq.Type.ERROR) {
2610 if (isTieBreak(response)) {
2611 this.outgoingContentAdd = null;
2612 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2613 return;
2614 }
2615 handleIqErrorResponse(response);
2616 }
2617 if (response.getType() == Iq.Type.TIMEOUT) {
2618 handleIqTimeoutResponse(response);
2619 }
2620 });
2621 }
2622
2623 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2624 if (isInitiator()) {
2625 this.initiatorRtpContentMap = rtpContentMap;
2626 } else {
2627 this.responderRtpContentMap = rtpContentMap;
2628 }
2629 }
2630
2631 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2632 if (isInitiator()) {
2633 this.responderRtpContentMap = rtpContentMap;
2634 } else {
2635 this.initiatorRtpContentMap = rtpContentMap;
2636 }
2637 }
2638
2639 // this method is to be used for content map modifications that modify media
2640 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2641 final RtpContentMap activeContents = rtpContentMap.activeContents();
2642 setLocalContentMap(activeContents);
2643 this.callIntegration.setAudioDeviceWhenAvailable(
2644 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2645 updateEndUserState();
2646 }
2647
2648 private SessionDescription setLocalSessionDescription()
2649 throws ExecutionException, InterruptedException {
2650 final org.webrtc.SessionDescription sessionDescription =
2651 this.webRTCWrapper.setLocalDescription(false).get();
2652 return SessionDescription.parse(sessionDescription.description);
2653 }
2654
2655 private void closeWebRTCSessionAfterFailedConnection() {
2656 this.webRTCWrapper.close();
2657 synchronized (this) {
2658 if (isTerminated()) {
2659 Log.d(
2660 Config.LOGTAG,
2661 id.account.getJid().asBareJid()
2662 + ": no need to send session-terminate after failed connection. Other party already did");
2663 return;
2664 }
2665 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2666 }
2667 }
2668
2669 public boolean zeroDuration() {
2670 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2671 }
2672
2673 public long getCallDuration() {
2674 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2675 }
2676
2677 @Override
2678 public CallIntegration getCallIntegration() {
2679 return this.callIntegration;
2680 }
2681
2682 public boolean isMicrophoneEnabled() {
2683 return webRTCWrapper.isMicrophoneEnabled();
2684 }
2685
2686 public boolean setMicrophoneEnabled(final boolean enabled) {
2687 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2688 }
2689
2690 public boolean isVideoEnabled() {
2691 return webRTCWrapper.isVideoEnabled();
2692 }
2693
2694 public void setVideoEnabled(final boolean enabled) {
2695 webRTCWrapper.setVideoEnabled(enabled);
2696 }
2697
2698 public boolean isCameraSwitchable() {
2699 return webRTCWrapper.isCameraSwitchable();
2700 }
2701
2702 public boolean isFrontCamera() {
2703 return webRTCWrapper.isFrontCamera();
2704 }
2705
2706 public ListenableFuture<Boolean> switchCamera() {
2707 return webRTCWrapper.switchCamera();
2708 }
2709
2710 @Override
2711 public synchronized void onCallIntegrationShowIncomingCallUi() {
2712 if (isTerminated()) {
2713 // there might be race conditions with the call integration service invoking this
2714 // callback when the rtp session has already ended.
2715 Log.w(
2716 Config.LOGTAG,
2717 "CallIntegration requested incoming call UI but session was already terminated");
2718 return;
2719 }
2720 // TODO apparently this can be called too early as well?
2721 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2722 }
2723
2724 @Override
2725 public void onCallIntegrationDisconnect() {
2726 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2727 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2728 rejectCall();
2729 } else {
2730 endCall();
2731 }
2732 }
2733
2734 @Override
2735 public void onCallIntegrationReject() {
2736 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2737 try {
2738 rejectCall();
2739 } catch (final IllegalStateException e) {
2740 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2741 }
2742 }
2743
2744 @Override
2745 public void onCallIntegrationAnswer() {
2746 // we need to start the UI to a) show it and b) be able to ask for permissions
2747 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2748 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2749 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2750 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2751 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2752 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2753 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2754 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2755 xmppConnectionService.startActivity(intent);
2756 }
2757
2758 @Override
2759 public void onCallIntegrationSilence() {
2760 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2761 }
2762
2763 @Override
2764 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2765 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2766 // that was previous disabled by the user. A proper implementation would probably be to
2767 // track user choice and enable the microphone with a userEnabled() &&
2768 // callIntegration.isMicrophoneEnabled() condition
2769 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2770 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2771 }
2772
2773 @Override
2774 public void onAudioDeviceChanged(
2775 final CallIntegration.AudioDevice selectedAudioDevice,
2776 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2777 Log.d(
2778 Config.LOGTAG,
2779 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2780 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2781 selectedAudioDevice, availableAudioDevices);
2782 }
2783
2784 private void updateEndUserState() {
2785 final RtpEndUserState endUserState = getEndUserState();
2786 this.updateCallIntegrationState();
2787 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2788 id.account, id.with, id.sessionId, endUserState);
2789 }
2790
2791 private void updateOngoingCallNotification() {
2792 final State state = this.state;
2793 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2794 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2795 && isResponder()) {
2796 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2797 xmppConnectionService.removeOngoingCall();
2798 return;
2799 }
2800 final boolean reconnecting;
2801 if (state == State.SESSION_ACCEPTED) {
2802 reconnecting =
2803 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2804 } else {
2805 reconnecting = false;
2806 }
2807 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2808 } else {
2809 xmppConnectionService.removeOngoingCall();
2810 }
2811 }
2812
2813 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2814 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2815 final Iq request = new Iq(Iq.Type.GET);
2816 request.setTo(id.account.getDomain());
2817 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2818 xmppConnectionService.sendIqPacket(
2819 id.account,
2820 request,
2821 (response) -> {
2822 final var iceServers = IceServers.parse(response);
2823 if (iceServers.isEmpty()) {
2824 Log.w(
2825 Config.LOGTAG,
2826 id.account.getJid().asBareJid()
2827 + ": no ICE server found "
2828 + response);
2829 }
2830 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2831 });
2832 } else {
2833 Log.w(
2834 Config.LOGTAG,
2835 id.account.getJid().asBareJid() + ": has no external service discovery");
2836 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2837 }
2838 }
2839
2840 @Override
2841 protected void terminateTransport() {
2842 this.webRTCWrapper.close();
2843 }
2844
2845 @Override
2846 protected void finish() {
2847 if (isTerminated()) {
2848 this.cancelRingingTimeout();
2849 this.callIntegration.verifyDisconnected();
2850 this.webRTCWrapper.verifyClosed();
2851 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2852 super.finish();
2853 try {
2854 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2855 log.getParentFile().mkdirs();
2856 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2857 } catch (final IOException e) { }
2858 } else {
2859 throw new IllegalStateException(
2860 String.format("Unable to call finish from %s", this.state));
2861 }
2862 }
2863
2864 private void writeLogMessage(final State state) {
2865 final long duration = getCallDuration();
2866 if (state == State.TERMINATED_SUCCESS
2867 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2868 writeLogMessageSuccess(duration);
2869 } else {
2870 writeLogMessageMissed();
2871 }
2872 }
2873
2874 private void writeLogMessageSuccess(final long duration) {
2875 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2876 this.writeMessage();
2877 }
2878
2879 private void writeLogMessageMissed() {
2880 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2881 this.writeMessage();
2882 }
2883
2884 private void writeMessage() {
2885 final Conversational conversational = message.getConversation();
2886 if (conversational instanceof Conversation) {
2887 ((Conversation) conversational).add(this.message);
2888 xmppConnectionService.createMessageAsync(message);
2889 xmppConnectionService.updateConversationUi();
2890 } else {
2891 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2892 }
2893 }
2894
2895 public Optional<VideoTrack> getLocalVideoTrack() {
2896 return webRTCWrapper.getLocalVideoTrack();
2897 }
2898
2899 public Optional<VideoTrack> getRemoteVideoTrack() {
2900 return webRTCWrapper.getRemoteVideoTrack();
2901 }
2902
2903 public EglBase.Context getEglBaseContext() {
2904 return webRTCWrapper.getEglBaseContext();
2905 }
2906
2907 void setProposedMedia(final Set<Media> media) {
2908 this.proposedMedia = media;
2909 this.callIntegration.setVideoState(
2910 Media.audioOnly(media)
2911 ? VideoProfile.STATE_AUDIO_ONLY
2912 : VideoProfile.STATE_BIDIRECTIONAL);
2913 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2914 }
2915
2916 public void fireStateUpdate() {
2917 final RtpEndUserState endUserState = getEndUserState();
2918 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2919 id.account, id.with, id.sessionId, endUserState);
2920 }
2921
2922 public boolean isSwitchToVideoAvailable() {
2923 final boolean prerequisite =
2924 Media.audioOnly(getMedia())
2925 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2926 .contains(getEndUserState());
2927 return prerequisite && remoteHasVideoFeature();
2928 }
2929
2930 private boolean remoteHasVideoFeature() {
2931 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2932 }
2933
2934 private boolean remoteHasSdpOfferAnswer() {
2935 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2936 }
2937
2938 @Override
2939 public Account getAccount() {
2940 return id.account;
2941 }
2942
2943 @Override
2944 public Jid getWith() {
2945 return id.with;
2946 }
2947
2948 @Override
2949 public String getSessionId() {
2950 return id.sessionId;
2951 }
2952
2953 private interface OnIceServersDiscovered {
2954 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2955 }
2956}