1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Account;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.services.CallIntegrationConnectionService;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import im.conversations.android.xmpp.model.disco.external.Services;
51import im.conversations.android.xmpp.model.jingle.Jingle;
52import im.conversations.android.xmpp.model.stanza.Iq;
53
54import org.webrtc.DtmfSender;
55import org.webrtc.EglBase;
56import org.webrtc.IceCandidate;
57import org.webrtc.PeerConnection;
58import org.webrtc.VideoTrack;
59
60import java.io.File;
61import java.io.IOException;
62import java.util.Arrays;
63import java.util.Collection;
64import java.util.Collections;
65import java.util.LinkedList;
66import java.util.List;
67import java.util.Map;
68import java.util.Queue;
69import java.util.Set;
70import java.util.concurrent.ExecutionException;
71import java.util.concurrent.ScheduledFuture;
72import java.util.concurrent.TimeUnit;
73import org.webrtc.EglBase;
74import org.webrtc.IceCandidate;
75import org.webrtc.PeerConnection;
76import org.webrtc.VideoTrack;
77
78public class JingleRtpConnection extends AbstractJingleConnection
79 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
80
81 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
82 public static final List<State> STATES_SHOWING_ONGOING_CALL =
83 Arrays.asList(
84 State.PROPOSED,
85 State.PROCEED,
86 State.SESSION_INITIALIZED_PRE_APPROVED,
87 State.SESSION_ACCEPTED);
88 private static final long BUSY_TIME_OUT = 30;
89
90 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
91 private final Queue<
92 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
93 pendingIceCandidates = new LinkedList<>();
94 private final OmemoVerification omemoVerification = new OmemoVerification();
95 public final CallIntegration callIntegration;
96 private final Message message;
97
98 private Set<Media> proposedMedia;
99 private RtpContentMap initiatorRtpContentMap;
100 private RtpContentMap responderRtpContentMap;
101 private RtpContentMap incomingContentAdd;
102 private RtpContentMap outgoingContentAdd;
103 private IceUdpTransportInfo.Setup peerDtlsSetup;
104 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
105 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
106 private ScheduledFuture<?> ringingTimeoutFuture;
107 private final long created = System.currentTimeMillis() / 1000L;
108
109 JingleRtpConnection(
110 final JingleConnectionManager jingleConnectionManager,
111 final Id id,
112 final Jid initiator) {
113 this(
114 jingleConnectionManager,
115 id,
116 initiator,
117 new CallIntegration(
118 jingleConnectionManager
119 .getXmppConnectionService()
120 .getApplicationContext()));
121 this.callIntegration.setAddress(
122 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
123 final var contact = id.getContact();
124 this.callIntegration.setCallerDisplayName(
125 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
126 this.callIntegration.setInitialized();
127 }
128
129 JingleRtpConnection(
130 final JingleConnectionManager jingleConnectionManager,
131 final Id id,
132 final Jid initiator,
133 final CallIntegration callIntegration) {
134 super(jingleConnectionManager, id, initiator);
135 final Conversation conversation =
136 jingleConnectionManager
137 .getXmppConnectionService()
138 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
139 this.message =
140 new Message(
141 conversation,
142 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
143 Message.TYPE_RTP_SESSION,
144 id.sessionId);
145 this.callIntegration = callIntegration;
146 this.callIntegration.setCallback(this);
147 }
148
149 @Override
150 synchronized void deliverPacket(final Iq iq) {
151 final var jingle = iq.getExtension(Jingle.class);
152 switch (jingle.getAction()) {
153 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
154 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
155 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
156 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
157 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
158 case CONTENT_ACCEPT -> receiveContentAccept(iq);
159 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
160 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
161 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
162 default -> {
163 respondOk(iq);
164 Log.d(
165 Config.LOGTAG,
166 String.format(
167 "%s: received unhandled jingle action %s",
168 id.account.getJid().asBareJid(), jingle.getAction()));
169 }
170 }
171 }
172
173 @Override
174 synchronized void notifyRebound() {
175 if (isTerminated()) {
176 return;
177 }
178 webRTCWrapper.close();
179 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
180 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
181 }
182 setExtraState("rebound");
183 if (isInState(
184 State.SESSION_INITIALIZED,
185 State.SESSION_INITIALIZED_PRE_APPROVED,
186 State.SESSION_ACCEPTED)) {
187 // we might have already changed resources (full jid) at this point; so this might not
188 // even reach the other party
189 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
190 } else {
191 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
192 finish();
193 }
194 }
195
196 public boolean applyDtmfTone(final String tone) {
197 return webRTCWrapper.applyDtmfTone(tone);
198 }
199
200 private void receiveSessionTerminate(final Iq jinglePacket) {
201 respondOk(jinglePacket);
202 final var jingle = jinglePacket.getExtension(Jingle.class);
203 final Jingle.ReasonWrapper wrapper = jingle.getReason();
204 final State previous = this.state;
205 Log.d(
206 Config.LOGTAG,
207 id.account.getJid().asBareJid()
208 + ": received session terminate reason="
209 + wrapper.reason
210 + "("
211 + Strings.nullToEmpty(wrapper.text)
212 + ") while in state "
213 + previous);
214 if (TERMINATED.contains(previous)) {
215 Log.d(
216 Config.LOGTAG,
217 id.account.getJid().asBareJid()
218 + ": ignoring session terminate because already in "
219 + previous);
220 return;
221 }
222 webRTCWrapper.close();
223 final State target = reasonToState(wrapper.reason);
224 transitionOrThrow(target);
225 writeLogMessage(target);
226 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
227 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
228 }
229 finish();
230 }
231
232 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
233 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
234 // INITIALIZED only after transport-info has been received
235 if (isInState(
236 State.NULL,
237 State.PROCEED,
238 State.SESSION_INITIALIZED,
239 State.SESSION_INITIALIZED_PRE_APPROVED,
240 State.SESSION_ACCEPTED)) {
241 final RtpContentMap contentMap;
242 try {
243 contentMap = RtpContentMap.of(jingle);
244 } catch (final IllegalArgumentException | NullPointerException e) {
245 Log.d(
246 Config.LOGTAG,
247 id.account.getJid().asBareJid()
248 + ": improperly formatted contents; ignoring",
249 e);
250 respondOk(jinglePacket);
251 return;
252 }
253 receiveTransportInfo(jinglePacket, contentMap);
254 } else {
255 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
256 }
257 }
258
259 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
260 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
261 candidates = contentMap.contents.entrySet();
262 final RtpContentMap remote = getRemoteContentMap();
263 final Set<String> remoteContentIds =
264 remote == null ? Collections.emptySet() : remote.contents.keySet();
265 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
266 Log.d(
267 Config.LOGTAG,
268 "received transport-info for unknown contents "
269 + contentMap.contents.keySet()
270 + " (known: "
271 + remoteContentIds
272 + ")");
273 respondOk(jinglePacket);
274 pendingIceCandidates.addAll(candidates);
275 return;
276 }
277 if (this.state != State.SESSION_ACCEPTED) {
278 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
279 respondOk(jinglePacket);
280 pendingIceCandidates.addAll(candidates);
281 return;
282 }
283 // zero candidates + modified credentials are an ICE restart offer
284 if (checkForIceRestart(jinglePacket, contentMap)) {
285 return;
286 }
287 respondOk(jinglePacket);
288 try {
289 processCandidates(candidates);
290 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
291 Log.w(
292 Config.LOGTAG,
293 id.account.getJid().asBareJid()
294 + ": PeerConnection was not initialized when processing transport info."
295 + " this usually indicates a race condition that can be ignored");
296 }
297 }
298
299 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
300 final RtpContentMap modification;
301 try {
302 modification = RtpContentMap.of(jingle);
303 modification.requireContentDescriptions();
304 } catch (final RuntimeException e) {
305 Log.d(
306 Config.LOGTAG,
307 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
308 Throwables.getRootCause(e));
309 respondOk(iq);
310 webRTCWrapper.close();
311 sendSessionTerminate(Reason.of(e), e.getMessage());
312 return;
313 }
314 if (isInState(State.SESSION_ACCEPTED)) {
315 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
316 final ListenableFuture<RtpContentMap> future =
317 receiveRtpContentMap(
318 modification,
319 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
320 Futures.addCallback(
321 future,
322 new FutureCallback<>() {
323 @Override
324 public void onSuccess(final RtpContentMap rtpContentMap) {
325 receiveContentAdd(iq, rtpContentMap);
326 }
327
328 @Override
329 public void onFailure(@NonNull Throwable throwable) {
330 respondOk(iq);
331 final Throwable rootCause = Throwables.getRootCause(throwable);
332 Log.d(
333 Config.LOGTAG,
334 id.account.getJid().asBareJid()
335 + ": improperly formatted contents in content-add",
336 throwable);
337 webRTCWrapper.close();
338 sendSessionTerminate(
339 Reason.ofThrowable(rootCause), rootCause.getMessage());
340 }
341 },
342 MoreExecutors.directExecutor());
343 } else {
344 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
345 }
346 }
347
348 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
349 final RtpContentMap remote = getRemoteContentMap();
350 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
351 respondOk(jinglePacket);
352 this.webRTCWrapper.close();
353 sendSessionTerminate(
354 Reason.FAILED_APPLICATION,
355 String.format(
356 "contents with names %s already exists",
357 Joiner.on(", ").join(modification.getNames())));
358 return;
359 }
360 final ContentAddition contentAddition =
361 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
362
363 final RtpContentMap outgoing = this.outgoingContentAdd;
364 final Set<ContentAddition.Summary> outgoingContentAddSummary =
365 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
366
367 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
368 if (isInitiator()) {
369 Log.d(
370 Config.LOGTAG,
371 id.getAccount().getJid().asBareJid()
372 + ": respond with tie break to matching content-add offer");
373 respondWithTieBreak(jinglePacket);
374 } else {
375 Log.d(
376 Config.LOGTAG,
377 id.getAccount().getJid().asBareJid()
378 + ": automatically accept matching content-add offer");
379 acceptContentAdd(contentAddition.summary, modification);
380 }
381 return;
382 }
383
384 // once we can display multiple video tracks we can be more loose with this condition
385 // theoretically it should also be fine to automatically accept audio only contents
386 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
387 Log.d(
388 Config.LOGTAG,
389 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
390 this.incomingContentAdd = modification;
391 respondOk(jinglePacket);
392 updateEndUserState();
393 } else {
394 respondOk(jinglePacket);
395 // TODO do we want to add a reason?
396 rejectContentAdd(modification);
397 }
398 }
399
400 private void receiveContentAccept(final Iq jinglePacket) {
401 final var jingle = jinglePacket.getExtension(Jingle.class);
402 final RtpContentMap receivedContentAccept;
403 try {
404 receivedContentAccept = RtpContentMap.of(jingle);
405 receivedContentAccept.requireContentDescriptions();
406 } catch (final RuntimeException e) {
407 Log.d(
408 Config.LOGTAG,
409 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
410 Throwables.getRootCause(e));
411 respondOk(jinglePacket);
412 webRTCWrapper.close();
413 sendSessionTerminate(Reason.of(e), e.getMessage());
414 return;
415 }
416
417 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
418 if (outgoingContentAdd == null) {
419 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
420 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
421 return;
422 }
423 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
424 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
425 this.outgoingContentAdd = null;
426 respondOk(jinglePacket);
427 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
428 final ListenableFuture<RtpContentMap> future =
429 receiveRtpContentMap(
430 receivedContentAccept,
431 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
432 Futures.addCallback(
433 future,
434 new FutureCallback<>() {
435 @Override
436 public void onSuccess(final RtpContentMap result) {
437 receiveContentAccept(result);
438 }
439
440 @Override
441 public void onFailure(@NonNull final Throwable throwable) {
442 webRTCWrapper.close();
443 sendSessionTerminate(
444 Reason.ofThrowable(throwable), throwable.getMessage());
445 }
446 },
447 MoreExecutors.directExecutor());
448 } else {
449 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
450 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
451 }
452 }
453
454 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
455 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
456 final RtpContentMap modifiedContentMap =
457 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
458
459 setRemoteContentMap(modifiedContentMap);
460
461 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
462
463 final org.webrtc.SessionDescription sdp =
464 new org.webrtc.SessionDescription(
465 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
466
467 try {
468 this.webRTCWrapper.setRemoteDescription(sdp).get();
469 } catch (final Exception e) {
470 final Throwable cause = Throwables.getRootCause(e);
471 Log.d(
472 Config.LOGTAG,
473 id.getAccount().getJid().asBareJid()
474 + ": unable to set remote description after receiving content-accept",
475 cause);
476 webRTCWrapper.close();
477 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
478 return;
479 }
480 Log.d(
481 Config.LOGTAG,
482 id.getAccount().getJid().asBareJid()
483 + ": remote has accepted content-add "
484 + ContentAddition.summary(receivedContentAccept));
485 processCandidates(receivedContentAccept.contents.entrySet());
486 updateEndUserState();
487 }
488
489 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
490 if (this.state != State.SESSION_ACCEPTED) {
491 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
492 return;
493 }
494 final Map<String, Content.Senders> modification =
495 Maps.transformEntries(
496 jingle.getJingleContents(), (key, value) -> value.getSenders());
497 final boolean isInitiator = isInitiator();
498 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
499 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
500 final Set<String> currentOutgoingMediaIds =
501 currentOutgoing == null
502 ? Collections.emptySet()
503 : currentOutgoing.contents.keySet();
504 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
505 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
506 respondOk(jinglePacket);
507 final RtpContentMap modifiedContentMap;
508 try {
509 modifiedContentMap =
510 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
511 } catch (final IllegalArgumentException e) {
512 webRTCWrapper.close();
513 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
514 return;
515 }
516 this.outgoingContentAdd = modifiedContentMap;
517 Log.d(
518 Config.LOGTAG,
519 id.account.getJid().asBareJid()
520 + ": processed content-modification for pending content-add");
521 } else if (remoteContentMap != null
522 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
523 respondOk(jinglePacket);
524 final RtpContentMap modifiedRemoteContentMap;
525 try {
526 modifiedRemoteContentMap =
527 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
528 } catch (final IllegalArgumentException e) {
529 webRTCWrapper.close();
530 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
531 return;
532 }
533 final SessionDescription offer;
534 try {
535 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
536 } catch (final IllegalArgumentException | NullPointerException e) {
537 Log.d(
538 Config.LOGTAG,
539 id.getAccount().getJid().asBareJid()
540 + ": unable convert offer from content-modify to SDP",
541 e);
542 webRTCWrapper.close();
543 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
544 return;
545 }
546 Log.d(
547 Config.LOGTAG,
548 id.account.getJid().asBareJid() + ": auto accepting content-modification");
549 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
550 } else {
551 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
552 respondWithItemNotFound(jinglePacket);
553 }
554 }
555
556 private void autoAcceptContentModify(
557 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
558 this.setRemoteContentMap(modifiedRemoteContentMap);
559 final org.webrtc.SessionDescription sdp =
560 new org.webrtc.SessionDescription(
561 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
562 try {
563 this.webRTCWrapper.setRemoteDescription(sdp).get();
564 // auto accept is only done when we already have tracks
565 final SessionDescription answer = setLocalSessionDescription();
566 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
567 modifyLocalContentMap(rtpContentMap);
568 // we do not need to send an answer but do we have to resend the candidates currently in
569 // SDP?
570 // resendCandidatesFromSdp(answer);
571 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
572 } catch (final Exception e) {
573 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
574 webRTCWrapper.close();
575 sendSessionTerminate(Reason.FAILED_APPLICATION);
576 }
577 }
578
579 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
580 final SessionDescription answer) {
581 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
582 new ImmutableMultimap.Builder<>();
583 for (final SessionDescription.Media media : answer.media) {
584 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
585 if (Strings.isNullOrEmpty(mid)) {
586 continue;
587 }
588 for (final String sdpCandidate : media.attributes.get("candidate")) {
589 final IceUdpTransportInfo.Candidate candidate =
590 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
591 if (candidate != null) {
592 candidateBuilder.put(mid, candidate);
593 }
594 }
595 }
596 return candidateBuilder.build();
597 }
598
599 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
600 final RtpContentMap receivedContentReject;
601 try {
602 receivedContentReject = RtpContentMap.of(jingle);
603 } catch (final RuntimeException e) {
604 Log.d(
605 Config.LOGTAG,
606 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
607 Throwables.getRootCause(e));
608 respondOk(jinglePacket);
609 this.webRTCWrapper.close();
610 sendSessionTerminate(Reason.of(e), e.getMessage());
611 return;
612 }
613
614 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
615 if (outgoingContentAdd == null) {
616 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
617 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
618 return;
619 }
620 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
621 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
622 this.outgoingContentAdd = null;
623 respondOk(jinglePacket);
624 Log.d(Config.LOGTAG, jinglePacket.toString());
625 receiveContentReject(ourSummary);
626 } else {
627 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
628 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
629 }
630 }
631
632 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
633 try {
634 this.webRTCWrapper.removeTrack(Media.VIDEO);
635 final RtpContentMap localContentMap = customRollback();
636 modifyLocalContentMap(localContentMap);
637 } catch (final Exception e) {
638 final Throwable cause = Throwables.getRootCause(e);
639 Log.d(
640 Config.LOGTAG,
641 id.getAccount().getJid().asBareJid()
642 + ": unable to rollback local description after receiving"
643 + " content-reject",
644 cause);
645 webRTCWrapper.close();
646 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
647 return;
648 }
649 Log.d(
650 Config.LOGTAG,
651 id.getAccount().getJid().asBareJid()
652 + ": remote has rejected our content-add "
653 + summary);
654 }
655
656 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
657 final RtpContentMap receivedContentRemove;
658 try {
659 receivedContentRemove = RtpContentMap.of(jingle);
660 receivedContentRemove.requireContentDescriptions();
661 } catch (final RuntimeException e) {
662 Log.d(
663 Config.LOGTAG,
664 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
665 Throwables.getRootCause(e));
666 respondOk(jinglePacket);
667 this.webRTCWrapper.close();
668 sendSessionTerminate(Reason.of(e), e.getMessage());
669 return;
670 }
671 respondOk(jinglePacket);
672 receiveContentRemove(receivedContentRemove);
673 }
674
675 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
676 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
677 final Set<ContentAddition.Summary> contentAddSummary =
678 incomingContentAdd == null
679 ? Collections.emptySet()
680 : ContentAddition.summary(incomingContentAdd);
681 final Set<ContentAddition.Summary> removeSummary =
682 ContentAddition.summary(receivedContentRemove);
683 if (contentAddSummary.equals(removeSummary)) {
684 this.incomingContentAdd = null;
685 updateEndUserState();
686 } else {
687 webRTCWrapper.close();
688 sendSessionTerminate(
689 Reason.FAILED_APPLICATION,
690 String.format(
691 "%s only supports %s as a means to retract a not yet accepted %s",
692 BuildConfig.APP_NAME,
693 Jingle.Action.CONTENT_REMOVE,
694 Jingle.Action.CONTENT_ADD));
695 }
696 }
697
698 public synchronized void retractContentAdd() {
699 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
700 if (outgoingContentAdd == null) {
701 throw new IllegalStateException("Not outgoing content add");
702 }
703 try {
704 webRTCWrapper.removeTrack(Media.VIDEO);
705 final RtpContentMap localContentMap = customRollback();
706 modifyLocalContentMap(localContentMap);
707 } catch (final Exception e) {
708 final Throwable cause = Throwables.getRootCause(e);
709 Log.d(
710 Config.LOGTAG,
711 id.getAccount().getJid().asBareJid()
712 + ": unable to rollback local description after trying to retract"
713 + " content-add",
714 cause);
715 webRTCWrapper.close();
716 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
717 return;
718 }
719 this.outgoingContentAdd = null;
720 final Iq retract =
721 outgoingContentAdd
722 .toStub()
723 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
724 this.send(retract);
725 Log.d(
726 Config.LOGTAG,
727 id.getAccount().getJid()
728 + ": retract content-add "
729 + ContentAddition.summary(outgoingContentAdd));
730 }
731
732 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
733 final SessionDescription sdp = setLocalSessionDescription();
734 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
735 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
736 this.webRTCWrapper
737 .setRemoteDescription(
738 new org.webrtc.SessionDescription(
739 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
740 .get();
741 return localRtpContentMap;
742 }
743
744 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
745 final RtpContentMap currentRemote = getRemoteContentMap();
746 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
747 if (diff.isEmpty()) {
748 throw new IllegalStateException(
749 "Unexpected rollback condition. No difference between local and remote");
750 }
751 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
752 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
753 final RtpContentMap nextRemote =
754 currentRemote.addContent(
755 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
756 return SessionDescription.of(nextRemote, isResponder());
757 }
758 throw new IllegalStateException(
759 "Unexpected rollback condition. Senders were not uniformly none");
760 }
761
762 public synchronized void acceptContentAdd(
763 @NonNull final Set<ContentAddition.Summary> contentAddition) {
764 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
765 if (incomingContentAdd == null) {
766 throw new IllegalStateException("No incoming content add");
767 }
768
769 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
770 this.incomingContentAdd = null;
771 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
772 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
773 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
774 Log.d(
775 Config.LOGTAG,
776 "content addition is receive only. we want to upgrade to 'both'");
777 final RtpContentMap modifiedSenders =
778 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
779 final Iq proposedContentModification =
780 modifiedSenders
781 .toStub()
782 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
783 proposedContentModification.setTo(id.with);
784 xmppConnectionService.sendIqPacket(
785 id.account,
786 proposedContentModification,
787 (response) -> {
788 if (response.getType() == Iq.Type.RESULT) {
789 Log.d(
790 Config.LOGTAG,
791 id.account.getJid().asBareJid()
792 + ": remote has accepted our upgrade to"
793 + " senders=both");
794 acceptContentAdd(
795 ContentAddition.summary(modifiedSenders), modifiedSenders);
796 } else {
797 Log.d(
798 Config.LOGTAG,
799 id.account.getJid().asBareJid()
800 + ": remote has rejected our upgrade to"
801 + " senders=both");
802 acceptContentAdd(contentAddition, incomingContentAdd);
803 }
804 });
805 } else {
806 acceptContentAdd(contentAddition, incomingContentAdd);
807 }
808 } else {
809 throw new IllegalStateException(
810 "Accepted content add does not match pending content-add");
811 }
812 }
813
814 private void acceptContentAdd(
815 @NonNull final Set<ContentAddition.Summary> contentAddition,
816 final RtpContentMap incomingContentAdd) {
817 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
818 final RtpContentMap modifiedContentMap =
819 getRemoteContentMap().addContent(incomingContentAdd, setup);
820 this.setRemoteContentMap(modifiedContentMap);
821
822 final SessionDescription offer;
823 try {
824 offer = SessionDescription.of(modifiedContentMap, isResponder());
825 } catch (final IllegalArgumentException | NullPointerException e) {
826 Log.d(
827 Config.LOGTAG,
828 id.getAccount().getJid().asBareJid()
829 + ": unable convert offer from content-add to SDP",
830 e);
831 webRTCWrapper.close();
832 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
833 return;
834 }
835 this.incomingContentAdd = null;
836 acceptContentAdd(contentAddition, offer);
837 }
838
839 private void acceptContentAdd(
840 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
841 final org.webrtc.SessionDescription sdp =
842 new org.webrtc.SessionDescription(
843 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
844 try {
845 this.webRTCWrapper.setRemoteDescription(sdp).get();
846
847 // TODO add tracks for 'media' where contentAddition.senders matches
848
849 // TODO if senders.sending(isInitiator())
850
851 this.webRTCWrapper.addTrack(Media.VIDEO);
852
853 // TODO add additional transceivers for recv only cases
854
855 final SessionDescription answer = setLocalSessionDescription();
856 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
857
858 final RtpContentMap contentAcceptMap =
859 rtpContentMap.toContentModification(
860 Collections2.transform(contentAddition, ca -> ca.name));
861
862 Log.d(
863 Config.LOGTAG,
864 id.getAccount().getJid().asBareJid()
865 + ": sending content-accept "
866 + ContentAddition.summary(contentAcceptMap));
867
868 addIceCandidatesFromBlackLog();
869
870 modifyLocalContentMap(rtpContentMap);
871 final ListenableFuture<RtpContentMap> future =
872 prepareOutgoingContentMap(contentAcceptMap);
873 Futures.addCallback(
874 future,
875 new FutureCallback<>() {
876 @Override
877 public void onSuccess(final RtpContentMap rtpContentMap) {
878 sendContentAccept(rtpContentMap);
879 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
880 }
881
882 @Override
883 public void onFailure(@NonNull final Throwable throwable) {
884 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
885 }
886 },
887 MoreExecutors.directExecutor());
888 } catch (final Exception e) {
889 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
890 webRTCWrapper.close();
891 sendSessionTerminate(Reason.FAILED_APPLICATION);
892 }
893 }
894
895 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
896 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
897 send(iq);
898 }
899
900 public synchronized void rejectContentAdd() {
901 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
902 if (incomingContentAdd == null) {
903 throw new IllegalStateException("No incoming content add");
904 }
905 this.incomingContentAdd = null;
906 updateEndUserState();
907 rejectContentAdd(incomingContentAdd);
908 }
909
910 private void rejectContentAdd(final RtpContentMap contentMap) {
911 final Iq iq =
912 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
913 Log.d(
914 Config.LOGTAG,
915 id.getAccount().getJid().asBareJid()
916 + ": rejecting content "
917 + ContentAddition.summary(contentMap));
918 send(iq);
919 }
920
921 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
922 final RtpContentMap existing = getRemoteContentMap();
923 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
924 final IceUdpTransportInfo.Credentials newCredentials;
925 try {
926 existingCredentials = existing.getCredentials();
927 newCredentials = rtpContentMap.getDistinctCredentials();
928 } catch (final IllegalStateException e) {
929 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
930 return false;
931 }
932 if (existingCredentials.contains(newCredentials)) {
933 return false;
934 }
935 // TODO an alternative approach is to check if we already got an iq result to our
936 // ICE-restart
937 // and if that's the case we are seeing an answer.
938 // This might be more spec compliant but also more error prone potentially
939 final boolean isSignalStateStable =
940 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
941 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
942 final boolean isOffer = rtpContentMap.emptyCandidates();
943 final RtpContentMap restartContentMap;
944 try {
945 if (isOffer) {
946 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
947 restartContentMap =
948 existing.modifiedCredentials(
949 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
950 } else {
951 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
952 Log.d(
953 Config.LOGTAG,
954 "received confirmation of ICE restart"
955 + newCredentials
956 + " peer_setup="
957 + setup);
958 // DTLS setup attribute needs to be rewritten to reflect current peer state
959 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
960 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
961 }
962 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
963 return isOffer;
964 } else {
965 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
966 respondWithTieBreak(jinglePacket);
967 return true;
968 }
969 } catch (final Exception exception) {
970 respondOk(jinglePacket);
971 final Throwable rootCause = Throwables.getRootCause(exception);
972 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
973 // If this happens a termination is already in progress
974 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
975 return true;
976 }
977 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
978 webRTCWrapper.close();
979 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
980 return true;
981 }
982 }
983
984 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
985 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
986 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
987 throw new IllegalStateException("Invalid peer setup");
988 }
989 return peerSetup;
990 }
991
992 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
993 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
994 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
995 }
996 this.peerDtlsSetup = setup;
997 }
998
999 private boolean applyIceRestart(
1000 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
1001 throws ExecutionException, InterruptedException {
1002 final SessionDescription sessionDescription =
1003 SessionDescription.of(restartContentMap, isResponder());
1004 final org.webrtc.SessionDescription.Type type =
1005 isOffer
1006 ? org.webrtc.SessionDescription.Type.OFFER
1007 : org.webrtc.SessionDescription.Type.ANSWER;
1008 org.webrtc.SessionDescription sdp =
1009 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1010 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1011 if (isInitiator()) {
1012 // We ignore the offer and respond with tie-break. This will clause the responder
1013 // not to apply the content map
1014 return false;
1015 }
1016 }
1017 webRTCWrapper.setRemoteDescription(sdp).get();
1018 setRemoteContentMap(restartContentMap);
1019 if (isOffer) {
1020 final SessionDescription localSessionDescription = setLocalSessionDescription();
1021 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1022 // We need to respond OK before sending any candidates
1023 respondOk(jinglePacket);
1024 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1025 } else {
1026 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1027 }
1028 return true;
1029 }
1030
1031 private void processCandidates(
1032 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1033 contents) {
1034 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1035 content : contents) {
1036 processCandidate(content);
1037 }
1038 }
1039
1040 private void processCandidate(
1041 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1042 content) {
1043 final RtpContentMap rtpContentMap = getRemoteContentMap();
1044 final List<String> indices = toIdentificationTags(rtpContentMap);
1045 final String sdpMid = content.getKey(); // aka content name
1046 final IceUdpTransportInfo transport = content.getValue().transport;
1047 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1048
1049 // TODO check that credentials remained the same
1050
1051 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1052 final String sdp;
1053 try {
1054 sdp = candidate.toSdpAttribute(credentials.ufrag);
1055 } catch (final IllegalArgumentException e) {
1056 Log.d(
1057 Config.LOGTAG,
1058 id.account.getJid().asBareJid()
1059 + ": ignoring invalid ICE candidate "
1060 + e.getMessage());
1061 continue;
1062 }
1063 final int mLineIndex = indices.indexOf(sdpMid);
1064 if (mLineIndex < 0) {
1065 Log.w(
1066 Config.LOGTAG,
1067 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1068 }
1069 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1070 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1071 this.webRTCWrapper.addIceCandidate(iceCandidate);
1072 }
1073 }
1074
1075 private RtpContentMap getRemoteContentMap() {
1076 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1077 }
1078
1079 private RtpContentMap getLocalContentMap() {
1080 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1081 }
1082
1083 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1084 final Group originalGroup = rtpContentMap.group;
1085 final List<String> identificationTags =
1086 originalGroup == null
1087 ? rtpContentMap.getNames()
1088 : originalGroup.getIdentificationTags();
1089 if (identificationTags.size() == 0) {
1090 Log.w(
1091 Config.LOGTAG,
1092 id.account.getJid().asBareJid()
1093 + ": no identification tags found in initial offer. we won't be able to"
1094 + " calculate mLineIndices");
1095 }
1096 return identificationTags;
1097 }
1098
1099 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1100 final Jingle jinglePacket, final boolean expectVerification) {
1101 try {
1102 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1103 } catch (final Exception e) {
1104 return Futures.immediateFailedFuture(e);
1105 }
1106 }
1107
1108 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1109 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1110 Log.d(
1111 Config.LOGTAG,
1112 "receiveRtpContentMap("
1113 + receivedContentMap.getClass().getSimpleName()
1114 + ",expectVerification="
1115 + expectVerification
1116 + ")");
1117 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1118 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1119 id.account
1120 .getAxolotlService()
1121 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1122 return Futures.transform(
1123 future,
1124 omemoVerifiedPayload -> {
1125 // TODO test if an exception here triggers a correct abort
1126 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1127 Log.d(
1128 Config.LOGTAG,
1129 id.account.getJid().asBareJid()
1130 + ": received verifiable DTLS fingerprint via "
1131 + omemoVerification);
1132 return omemoVerifiedPayload.getPayload();
1133 },
1134 MoreExecutors.directExecutor());
1135 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1136 return Futures.immediateFailedFuture(
1137 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1138 } else {
1139 return Futures.immediateFuture(receivedContentMap);
1140 }
1141 }
1142
1143 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1144 if (isInitiator()) {
1145 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1146 return;
1147 }
1148 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1149 Futures.addCallback(
1150 future,
1151 new FutureCallback<>() {
1152 @Override
1153 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1154 receiveSessionInitiate(jinglePacket, rtpContentMap);
1155 }
1156
1157 @Override
1158 public void onFailure(@NonNull final Throwable throwable) {
1159 respondOk(jinglePacket);
1160 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1161 }
1162 },
1163 MoreExecutors.directExecutor());
1164 }
1165
1166 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1167 try {
1168 contentMap.requireContentDescriptions();
1169 contentMap.requireDTLSFingerprint(true);
1170 } catch (final RuntimeException e) {
1171 Log.d(
1172 Config.LOGTAG,
1173 id.account.getJid().asBareJid() + ": improperly formatted contents",
1174 Throwables.getRootCause(e));
1175 respondOk(jinglePacket);
1176 sendSessionTerminate(Reason.of(e), e.getMessage());
1177 return;
1178 }
1179 Log.d(
1180 Config.LOGTAG,
1181 "processing session-init with " + contentMap.contents.size() + " contents");
1182 final State target;
1183 if (this.state == State.PROCEED) {
1184 Preconditions.checkState(
1185 proposedMedia != null && proposedMedia.size() > 0,
1186 "proposed media must be set when processing pre-approved session-initiate");
1187 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1188 sendSessionTerminate(
1189 Reason.SECURITY_ERROR,
1190 String.format(
1191 "Your session proposal (Jingle Message Initiation) included media"
1192 + " %s but your session-initiate was %s",
1193 this.proposedMedia, contentMap.getMedia()));
1194 return;
1195 }
1196 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1197 } else {
1198 target = State.SESSION_INITIALIZED;
1199 setProposedMedia(contentMap.getMedia());
1200 }
1201 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1202 respondOk(jinglePacket);
1203 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1204 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1205 Log.d(
1206 Config.LOGTAG,
1207 id.account.getJid().asBareJid()
1208 + ": automatically accepting session-initiate");
1209 sendSessionAccept();
1210 } else {
1211 Log.d(
1212 Config.LOGTAG,
1213 id.account.getJid().asBareJid()
1214 + ": received not pre-approved session-initiate. start ringing");
1215 startRinging();
1216 }
1217 } else {
1218 Log.d(
1219 Config.LOGTAG,
1220 String.format(
1221 "%s: received session-initiate while in state %s",
1222 id.account.getJid().asBareJid(), state));
1223 setExtraState("out of order");
1224 terminateWithOutOfOrder(jinglePacket);
1225 }
1226 }
1227
1228 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1229 if (isResponder()) {
1230 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1231 return;
1232 }
1233 final ListenableFuture<RtpContentMap> future =
1234 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1235 Futures.addCallback(
1236 future,
1237 new FutureCallback<>() {
1238 @Override
1239 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1240 receiveSessionAccept(jinglePacket, rtpContentMap);
1241 }
1242
1243 @Override
1244 public void onFailure(@NonNull final Throwable throwable) {
1245 respondOk(jinglePacket);
1246 Log.d(
1247 Config.LOGTAG,
1248 id.account.getJid().asBareJid()
1249 + ": improperly formatted contents in session-accept",
1250 throwable);
1251 webRTCWrapper.close();
1252 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1253 }
1254 },
1255 MoreExecutors.directExecutor());
1256 }
1257
1258 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1259 try {
1260 contentMap.requireContentDescriptions();
1261 contentMap.requireDTLSFingerprint();
1262 } catch (final RuntimeException e) {
1263 setExtraState("bad contents");
1264 respondOk(jinglePacket);
1265 Log.d(
1266 Config.LOGTAG,
1267 id.account.getJid().asBareJid()
1268 + ": improperly formatted contents in session-accept",
1269 e);
1270 webRTCWrapper.close();
1271 sendSessionTerminate(Reason.of(e), e.getMessage());
1272 return;
1273 }
1274 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1275 if (!initiatorMedia.equals(contentMap.getMedia())) {
1276 sendSessionTerminate(
1277 Reason.SECURITY_ERROR,
1278 String.format(
1279 "Your session-included included media %s but our session-initiate was"
1280 + " %s",
1281 this.proposedMedia, contentMap.getMedia()));
1282 return;
1283 }
1284 Log.d(
1285 Config.LOGTAG,
1286 "processing session-accept with " + contentMap.contents.size() + " contents");
1287 if (transition(State.SESSION_ACCEPTED)) {
1288 respondOk(jinglePacket);
1289 receiveSessionAccept(contentMap);
1290 } else {
1291 Log.d(
1292 Config.LOGTAG,
1293 String.format(
1294 "%s: received session-accept while in state %s",
1295 id.account.getJid().asBareJid(), state));
1296 respondOk(jinglePacket);
1297 }
1298 }
1299
1300 private void receiveSessionAccept(final RtpContentMap contentMap) {
1301 this.responderRtpContentMap = contentMap;
1302 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1303 final SessionDescription sessionDescription;
1304 try {
1305 sessionDescription = SessionDescription.of(contentMap, false);
1306 } catch (final IllegalArgumentException | NullPointerException e) {
1307 Log.d(
1308 Config.LOGTAG,
1309 id.account.getJid().asBareJid()
1310 + ": unable convert offer from session-accept to SDP",
1311 e);
1312 webRTCWrapper.close();
1313 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1314 return;
1315 }
1316 final org.webrtc.SessionDescription answer =
1317 new org.webrtc.SessionDescription(
1318 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1319 try {
1320 this.webRTCWrapper.setRemoteDescription(answer).get();
1321 } catch (final Exception e) {
1322 Log.d(
1323 Config.LOGTAG,
1324 id.account.getJid().asBareJid()
1325 + ": unable to set remote description after receiving session-accept",
1326 Throwables.getRootCause(e));
1327 webRTCWrapper.close();
1328 sendSessionTerminate(
1329 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1330 return;
1331 }
1332 processCandidates(contentMap.contents.entrySet());
1333 }
1334
1335 private void sendSessionAccept() {
1336 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1337 if (rtpContentMap == null) {
1338 throw new IllegalStateException("initiator RTP Content Map has not been set");
1339 }
1340 final SessionDescription offer;
1341 try {
1342 offer = SessionDescription.of(rtpContentMap, true);
1343 } catch (final IllegalArgumentException | NullPointerException e) {
1344 Log.d(
1345 Config.LOGTAG,
1346 id.account.getJid().asBareJid()
1347 + ": unable convert offer from session-initiate to SDP",
1348 e);
1349 webRTCWrapper.close();
1350 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1351 return;
1352 }
1353 sendSessionAccept(rtpContentMap.getMedia(), offer);
1354 }
1355
1356 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1357 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1358 }
1359
1360 private synchronized void sendSessionAccept(
1361 final Set<Media> media,
1362 final SessionDescription offer,
1363 final Collection<PeerConnection.IceServer> iceServers) {
1364 if (isTerminated()) {
1365 Log.w(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid()
1368 + ": ICE servers got discovered when session was already terminated."
1369 + " nothing to do.");
1370 return;
1371 }
1372 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1373 try {
1374 setupWebRTC(media, iceServers, !includeCandidates);
1375 } catch (final WebRTCWrapper.InitializationException e) {
1376 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1377 webRTCWrapper.close();
1378 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1379 return;
1380 }
1381 final org.webrtc.SessionDescription sdp =
1382 new org.webrtc.SessionDescription(
1383 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1384 try {
1385 this.webRTCWrapper.setRemoteDescription(sdp).get();
1386 addIceCandidatesFromBlackLog();
1387 org.webrtc.SessionDescription webRTCSessionDescription =
1388 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1389 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1390 } catch (final Exception e) {
1391 failureToAcceptSession(e);
1392 }
1393 }
1394
1395 private void failureToAcceptSession(final Throwable throwable) {
1396 if (isTerminated()) {
1397 return;
1398 }
1399 final Throwable rootCause = Throwables.getRootCause(throwable);
1400 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1401 webRTCWrapper.close();
1402 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1403 }
1404
1405 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1406 if (isTerminated()) {
1407 return;
1408 }
1409 final Throwable rootCause = Throwables.getRootCause(throwable);
1410 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1411 webRTCWrapper.close();
1412 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1413 }
1414
1415 private void addIceCandidatesFromBlackLog() {
1416 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1417 while ((foo = this.pendingIceCandidates.poll()) != null) {
1418 processCandidate(foo);
1419 Log.d(
1420 Config.LOGTAG,
1421 id.account.getJid().asBareJid() + ": added candidate from back log");
1422 }
1423 }
1424
1425 private void prepareSessionAccept(
1426 final org.webrtc.SessionDescription webRTCSessionDescription,
1427 final boolean includeCandidates) {
1428 final SessionDescription sessionDescription =
1429 SessionDescription.parse(webRTCSessionDescription.description);
1430 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1431 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1432 if (includeCandidates) {
1433 candidates = parseCandidates(sessionDescription);
1434 } else {
1435 candidates = ImmutableMultimap.of();
1436 }
1437 this.responderRtpContentMap = respondingRtpContentMap;
1438 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1439 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1440 prepareOutgoingContentMap(respondingRtpContentMap);
1441 Futures.addCallback(
1442 outgoingContentMapFuture,
1443 new FutureCallback<>() {
1444 @Override
1445 public void onSuccess(final RtpContentMap outgoingContentMap) {
1446 if (includeCandidates) {
1447 Log.d(
1448 Config.LOGTAG,
1449 "including "
1450 + candidates.size()
1451 + " candidates in session accept");
1452 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1453 } else {
1454 sendSessionAccept(outgoingContentMap);
1455 }
1456 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1457 }
1458
1459 @Override
1460 public void onFailure(@NonNull Throwable throwable) {
1461 failureToAcceptSession(throwable);
1462 }
1463 },
1464 MoreExecutors.directExecutor());
1465 }
1466
1467 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1468 if (isTerminated()) {
1469 Log.w(
1470 Config.LOGTAG,
1471 id.account.getJid().asBareJid()
1472 + ": preparing session accept was too slow. already terminated. nothing"
1473 + " to do.");
1474 return;
1475 }
1476 transitionOrThrow(State.SESSION_ACCEPTED);
1477 final Iq sessionAccept =
1478 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1479 send(sessionAccept);
1480 }
1481
1482 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1483 final RtpContentMap rtpContentMap) {
1484 if (this.omemoVerification.hasDeviceId()) {
1485 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1486 verifiedPayloadFuture =
1487 id.account
1488 .getAxolotlService()
1489 .encrypt(
1490 rtpContentMap,
1491 id.with,
1492 omemoVerification.getDeviceId());
1493 return Futures.transform(
1494 verifiedPayloadFuture,
1495 verifiedPayload -> {
1496 omemoVerification.setOrEnsureEqual(verifiedPayload);
1497 return verifiedPayload.getPayload();
1498 },
1499 MoreExecutors.directExecutor());
1500 } else {
1501 return Futures.immediateFuture(rtpContentMap);
1502 }
1503 }
1504
1505 synchronized void deliveryMessage(
1506 final Jid from,
1507 final Element message,
1508 final String serverMessageId,
1509 final long timestamp) {
1510 Log.d(
1511 Config.LOGTAG,
1512 id.account.getJid().asBareJid()
1513 + ": delivered message to JingleRtpConnection "
1514 + message);
1515 switch (message.getName()) {
1516 case "propose" ->
1517 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1518 case "proceed" ->
1519 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1520 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1521 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1522 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1523 }
1524 }
1525
1526 void deliverFailedProceed(final String message) {
1527 Log.d(
1528 Config.LOGTAG,
1529 id.account.getJid().asBareJid()
1530 + ": receive message error for proceed message ("
1531 + Strings.nullToEmpty(message)
1532 + ")");
1533 setExtraState("proceed error");
1534 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1535 webRTCWrapper.close();
1536 Log.d(
1537 Config.LOGTAG,
1538 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1539 this.finish();
1540 }
1541 }
1542
1543 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1544 final boolean originatedFromMyself =
1545 from.asBareJid().equals(id.account.getJid().asBareJid());
1546 if (originatedFromMyself) {
1547 if (transition(State.ACCEPTED)) {
1548 acceptedOnOtherDevice(serverMsgId, timestamp);
1549 } else {
1550 Log.d(
1551 Config.LOGTAG,
1552 id.account.getJid().asBareJid()
1553 + ": unable to transition to accept because already in state="
1554 + this.state);
1555 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1556 }
1557 } else {
1558 Log.d(
1559 Config.LOGTAG,
1560 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1561 }
1562 }
1563
1564 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1565 if (serverMsgId != null) {
1566 this.message.setServerMsgId(serverMsgId);
1567 }
1568 this.message.setTime(timestamp);
1569 this.message.setCarbon(true); // indicate that call was accepted on other device
1570 this.writeLogMessageSuccess(0);
1571 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1572 this.finish();
1573 }
1574
1575 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1576 final boolean originatedFromMyself =
1577 from.asBareJid().equals(id.account.getJid().asBareJid());
1578 // reject from another one of my clients
1579 if (originatedFromMyself) {
1580 receiveRejectFromMyself(serverMsgId, timestamp);
1581 } else if (isInitiator()) {
1582 if (from.equals(id.with)) {
1583 receiveRejectFromResponder();
1584 } else {
1585 Log.d(
1586 Config.LOGTAG,
1587 id.account.getJid()
1588 + ": ignoring reject from "
1589 + from
1590 + " for session with "
1591 + id.with);
1592 }
1593 } else {
1594 Log.d(
1595 Config.LOGTAG,
1596 id.account.getJid()
1597 + ": ignoring reject from "
1598 + from
1599 + " for session with "
1600 + id.with);
1601 }
1602 }
1603
1604 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1605 setExtraState("rejected");
1606 if (transition(State.REJECTED)) {
1607 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1608 this.finish();
1609 if (serverMsgId != null) {
1610 this.message.setServerMsgId(serverMsgId);
1611 }
1612 this.message.setTime(timestamp);
1613 this.message.setCarbon(true); // indicate that call was rejected on other device
1614 writeLogMessageMissed();
1615 } else {
1616 Log.d(
1617 Config.LOGTAG,
1618 "not able to transition into REJECTED because already in " + this.state);
1619 }
1620 }
1621
1622 private void receiveRejectFromResponder() {
1623 setExtraState("responder rejected");
1624 if (isInState(State.PROCEED)) {
1625 Log.d(
1626 Config.LOGTAG,
1627 id.account.getJid()
1628 + ": received reject while still in proceed. callee reconsidered");
1629 closeTransitionLogFinish(State.REJECTED_RACED);
1630 return;
1631 }
1632 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1633 Log.d(
1634 Config.LOGTAG,
1635 id.account.getJid()
1636 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee"
1637 + " reconsidered before receiving session-init");
1638 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1639 return;
1640 }
1641 Log.d(
1642 Config.LOGTAG,
1643 id.account.getJid()
1644 + ": ignoring reject from responder because already in state "
1645 + this.state);
1646 }
1647
1648 private void receivePropose(
1649 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1650 final boolean originatedFromMyself =
1651 from.asBareJid().equals(id.account.getJid().asBareJid());
1652 if (originatedFromMyself) {
1653 Log.d(
1654 Config.LOGTAG,
1655 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1656 } else if (transition(
1657 State.PROPOSED,
1658 () -> {
1659 final Collection<RtpDescription> descriptions =
1660 Collections2.transform(
1661 Collections2.filter(
1662 propose.getDescriptions(),
1663 d -> d instanceof RtpDescription),
1664 input -> (RtpDescription) input);
1665 final Collection<Media> media =
1666 Collections2.transform(descriptions, RtpDescription::getMedia);
1667 Preconditions.checkState(
1668 !media.contains(Media.UNKNOWN),
1669 "RTP descriptions contain unknown media");
1670 Log.d(
1671 Config.LOGTAG,
1672 id.account.getJid().asBareJid()
1673 + ": received session proposal from "
1674 + from
1675 + " for "
1676 + media);
1677 this.setProposedMedia(Sets.newHashSet(media));
1678 })) {
1679 if (serverMsgId != null) {
1680 this.message.setServerMsgId(serverMsgId);
1681 }
1682 this.message.setTime(timestamp);
1683 startRinging();
1684 // in environments where we always use discovery timeouts we always want to respond with
1685 // 'ringing'
1686 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1687 || id.getContact().showInContactList()) {
1688 sendJingleMessage("ringing");
1689 }
1690 } else {
1691 Log.d(
1692 Config.LOGTAG,
1693 id.account.getJid()
1694 + ": ignoring session proposal because already in "
1695 + state);
1696 }
1697 }
1698
1699 private void startRinging() {
1700 this.callIntegration.setRinging();
1701 Log.d(
1702 Config.LOGTAG,
1703 id.account.getJid().asBareJid()
1704 + ": received call from "
1705 + id.with
1706 + ". start ringing");
1707 ringingTimeoutFuture =
1708 jingleConnectionManager.schedule(
1709 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1710 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1711 return;
1712 }
1713 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1714 }
1715
1716 private synchronized void ringingTimeout() {
1717 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1718 switch (this.state) {
1719 case PROPOSED -> {
1720 message.markUnread();
1721 rejectCallFromProposed();
1722 }
1723 case SESSION_INITIALIZED -> {
1724 message.markUnread();
1725 rejectCallFromSessionInitiate();
1726 }
1727 }
1728 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1729 }
1730
1731 private void cancelRingingTimeout() {
1732 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1733 if (future != null && !future.isCancelled()) {
1734 future.cancel(false);
1735 }
1736 }
1737
1738 private void receiveProceed(
1739 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1740 final Set<Media> media =
1741 Preconditions.checkNotNull(
1742 this.proposedMedia, "Proposed media has to be set before handling proceed");
1743 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1744 if (from.equals(id.with)) {
1745 if (isInitiator()) {
1746 if (transition(State.PROCEED)) {
1747 if (serverMsgId != null) {
1748 this.message.setServerMsgId(serverMsgId);
1749 }
1750 this.message.setTime(timestamp);
1751 final Integer remoteDeviceId = proceed.getDeviceId();
1752 if (isOmemoEnabled()) {
1753 this.omemoVerification.setDeviceId(remoteDeviceId);
1754 } else {
1755 if (remoteDeviceId != null) {
1756 Log.d(
1757 Config.LOGTAG,
1758 id.account.getJid().asBareJid()
1759 + ": remote party signaled support for OMEMO"
1760 + " verification but we have OMEMO disabled");
1761 }
1762 this.omemoVerification.setDeviceId(null);
1763 }
1764 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1765 } else {
1766 Log.d(
1767 Config.LOGTAG,
1768 String.format(
1769 "%s: ignoring proceed because already in %s",
1770 id.account.getJid().asBareJid(), this.state));
1771 }
1772 } else {
1773 Log.d(
1774 Config.LOGTAG,
1775 String.format(
1776 "%s: ignoring proceed because we were not initializing",
1777 id.account.getJid().asBareJid()));
1778 }
1779 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1780 if (transition(State.ACCEPTED)) {
1781 Log.d(
1782 Config.LOGTAG,
1783 id.account.getJid().asBareJid()
1784 + ": moved session with "
1785 + id.with
1786 + " into state accepted after received carbon copied proceed");
1787 acceptedOnOtherDevice(serverMsgId, timestamp);
1788 }
1789 } else {
1790 Log.d(
1791 Config.LOGTAG,
1792 String.format(
1793 "%s: ignoring proceed from %s. was expected from %s",
1794 id.account.getJid().asBareJid(), from, id.with));
1795 }
1796 }
1797
1798 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1799 if (from.equals(id.with)) {
1800 final State target =
1801 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1802 if (transition(target)) {
1803 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1804 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1805 Log.d(
1806 Config.LOGTAG,
1807 id.account.getJid().asBareJid()
1808 + ": session with "
1809 + id.with
1810 + " has been retracted (serverMsgId="
1811 + serverMsgId
1812 + ")");
1813 if (serverMsgId != null) {
1814 this.message.setServerMsgId(serverMsgId);
1815 }
1816 this.message.setTime(timestamp);
1817 if (target == State.RETRACTED) {
1818 this.message.markUnread();
1819 }
1820 writeLogMessageMissed();
1821 finish();
1822 } else {
1823 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1824 }
1825 } else {
1826 // TODO parse retract from self
1827 Log.d(
1828 Config.LOGTAG,
1829 id.account.getJid().asBareJid()
1830 + ": received retract from "
1831 + from
1832 + ". expected retract from"
1833 + id.with
1834 + ". ignoring");
1835 }
1836 }
1837
1838 public void sendSessionInitiate() {
1839 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1840 }
1841
1842 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1843 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1844 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1845 }
1846
1847 private synchronized void sendSessionInitiate(
1848 final Set<Media> media,
1849 final State targetState,
1850 final Collection<PeerConnection.IceServer> iceServers) {
1851 if (isTerminated()) {
1852 Log.w(
1853 Config.LOGTAG,
1854 id.account.getJid().asBareJid()
1855 + ": ICE servers got discovered when session was already terminated."
1856 + " nothing to do.");
1857 return;
1858 }
1859 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1860 try {
1861 setupWebRTC(media, iceServers, !includeCandidates);
1862 } catch (final WebRTCWrapper.InitializationException e) {
1863 setExtraState("Unable to init WebRTC");
1864 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1865 webRTCWrapper.close();
1866 sendRetract(Reason.ofThrowable(e));
1867 return;
1868 }
1869 try {
1870 org.webrtc.SessionDescription webRTCSessionDescription =
1871 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1872 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1873 } catch (final Exception e) {
1874 setExtraState("Unable to setLocalDescription");
1875 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1876 // exceptions
1877 failureToInitiateSession(e, targetState);
1878 }
1879 }
1880
1881 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1882 if (isTerminated()) {
1883 return;
1884 }
1885 Log.d(
1886 Config.LOGTAG,
1887 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1888 Throwables.getRootCause(throwable));
1889 webRTCWrapper.close();
1890 final Reason reason = Reason.ofThrowable(throwable);
1891 if (isInState(targetState)) {
1892 sendSessionTerminate(reason, throwable.getMessage());
1893 } else {
1894 sendRetract(reason);
1895 }
1896 }
1897
1898 private void sendRetract(final Reason reason) {
1899 // TODO embed reason into retract
1900 sendJingleMessage("retract", id.with.asBareJid());
1901 transitionOrThrow(reasonToState(reason));
1902 this.finish();
1903 }
1904
1905 private void prepareSessionInitiate(
1906 final org.webrtc.SessionDescription webRTCSessionDescription,
1907 final boolean includeCandidates,
1908 final State targetState) {
1909 final SessionDescription sessionDescription =
1910 SessionDescription.parse(webRTCSessionDescription.description);
1911 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1912 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1913 if (includeCandidates) {
1914 candidates = parseCandidates(sessionDescription);
1915 } else {
1916 candidates = ImmutableMultimap.of();
1917 }
1918 this.initiatorRtpContentMap = rtpContentMap;
1919 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1920 encryptSessionInitiate(rtpContentMap);
1921 Futures.addCallback(
1922 outgoingContentMapFuture,
1923 new FutureCallback<>() {
1924 @Override
1925 public void onSuccess(final RtpContentMap outgoingContentMap) {
1926 if (includeCandidates) {
1927 Log.d(
1928 Config.LOGTAG,
1929 "including "
1930 + candidates.size()
1931 + " candidates in session initiate");
1932 sendSessionInitiate(
1933 outgoingContentMap.withCandidates(candidates), targetState);
1934 } else {
1935 sendSessionInitiate(outgoingContentMap, targetState);
1936 }
1937 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1938 }
1939
1940 @Override
1941 public void onFailure(@NonNull final Throwable throwable) {
1942 setExtraState("Unable to outgoingContentMapFuture");
1943 failureToInitiateSession(throwable, targetState);
1944 }
1945 },
1946 MoreExecutors.directExecutor());
1947 }
1948
1949 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1950 if (isTerminated()) {
1951 Log.w(
1952 Config.LOGTAG,
1953 id.account.getJid().asBareJid()
1954 + ": preparing session was too slow. already terminated. nothing to"
1955 + " do.");
1956 return;
1957 }
1958 this.transitionOrThrow(targetState);
1959 final Iq sessionInitiate =
1960 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1961 send(sessionInitiate);
1962 }
1963
1964 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1965 final RtpContentMap rtpContentMap) {
1966 if (this.omemoVerification.hasDeviceId()) {
1967 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1968 verifiedPayloadFuture =
1969 id.account
1970 .getAxolotlService()
1971 .encrypt(
1972 rtpContentMap,
1973 id.with,
1974 omemoVerification.getDeviceId());
1975 final ListenableFuture<RtpContentMap> future =
1976 Futures.transform(
1977 verifiedPayloadFuture,
1978 verifiedPayload -> {
1979 omemoVerification.setSessionFingerprint(
1980 verifiedPayload.getFingerprint());
1981 return verifiedPayload.getPayload();
1982 },
1983 MoreExecutors.directExecutor());
1984 if (Config.REQUIRE_RTP_VERIFICATION) {
1985 return future;
1986 }
1987 return Futures.catching(
1988 future,
1989 CryptoFailedException.class,
1990 e -> {
1991 Log.w(
1992 Config.LOGTAG,
1993 id.account.getJid().asBareJid()
1994 + ": unable to use OMEMO DTLS verification on outgoing"
1995 + " session initiate. falling back",
1996 e);
1997 return rtpContentMap;
1998 },
1999 MoreExecutors.directExecutor());
2000 } else {
2001 return Futures.immediateFuture(rtpContentMap);
2002 }
2003 }
2004
2005 protected void sendSessionTerminate(final Reason reason) {
2006 sendSessionTerminate(reason, null);
2007 }
2008
2009 protected void sendSessionTerminate(final Reason reason, final String text) {
2010 sendSessionTerminate(reason, text, this::writeLogMessage);
2011 sendJingleMessageFinish(reason);
2012 }
2013
2014 private void sendTransportInfo(
2015 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2016 final RtpContentMap transportInfo;
2017 try {
2018 final RtpContentMap rtpContentMap =
2019 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2020 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2021 } catch (final Exception e) {
2022 Log.d(
2023 Config.LOGTAG,
2024 id.account.getJid().asBareJid()
2025 + ": unable to prepare transport-info from candidate for content="
2026 + contentName);
2027 return;
2028 }
2029 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2030 send(iq);
2031 }
2032
2033 public RtpEndUserState getEndUserState() {
2034 switch (this.state) {
2035 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2036 if (isInitiator()) {
2037 return RtpEndUserState.RINGING;
2038 } else {
2039 return RtpEndUserState.INCOMING_CALL;
2040 }
2041 }
2042 case PROCEED -> {
2043 if (isInitiator()) {
2044 return RtpEndUserState.RINGING;
2045 } else {
2046 return RtpEndUserState.ACCEPTING_CALL;
2047 }
2048 }
2049 case SESSION_INITIALIZED_PRE_APPROVED -> {
2050 if (isInitiator()) {
2051 return RtpEndUserState.RINGING;
2052 } else {
2053 return RtpEndUserState.CONNECTING;
2054 }
2055 }
2056 case SESSION_ACCEPTED -> {
2057 final ContentAddition ca = getPendingContentAddition();
2058 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2059 return RtpEndUserState.INCOMING_CONTENT_ADD;
2060 }
2061 return getPeerConnectionStateAsEndUserState();
2062 }
2063 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2064 if (isInitiator()) {
2065 return RtpEndUserState.DECLINED_OR_BUSY;
2066 } else {
2067 return RtpEndUserState.ENDED;
2068 }
2069 }
2070 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2071 return RtpEndUserState.ENDED;
2072 }
2073 case RETRACTED_RACED -> {
2074 if (isInitiator()) {
2075 return RtpEndUserState.ENDED;
2076 } else {
2077 return RtpEndUserState.RETRACTED;
2078 }
2079 }
2080 case TERMINATED_CONNECTIVITY_ERROR -> {
2081 return zeroDuration()
2082 ? RtpEndUserState.CONNECTIVITY_ERROR
2083 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2084 }
2085 case TERMINATED_APPLICATION_FAILURE -> {
2086 return RtpEndUserState.APPLICATION_ERROR;
2087 }
2088 case TERMINATED_SECURITY_ERROR -> {
2089 return RtpEndUserState.SECURITY_ERROR;
2090 }
2091 }
2092 throw new IllegalStateException(
2093 String.format("%s has no equivalent EndUserState", this.state));
2094 }
2095
2096 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2097 final PeerConnection.PeerConnectionState state;
2098 try {
2099 state = webRTCWrapper.getState();
2100 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2101 // We usually close the WebRTCWrapper *before* transitioning so we might still
2102 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2103 return RtpEndUserState.ENDING_CALL;
2104 }
2105 return switch (state) {
2106 case CONNECTED -> RtpEndUserState.CONNECTED;
2107 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2108 case CLOSED -> RtpEndUserState.ENDING_CALL;
2109 default ->
2110 zeroDuration()
2111 ? RtpEndUserState.CONNECTIVITY_ERROR
2112 : RtpEndUserState.RECONNECTING;
2113 };
2114 }
2115
2116 private boolean isPeerConnectionConnected() {
2117 try {
2118 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2119 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2120 return false;
2121 }
2122 }
2123
2124 private void updateCallIntegrationState() {
2125 switch (this.state) {
2126 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2127 if (isInitiator()) {
2128 this.callIntegration.setDialing();
2129 } else {
2130 this.callIntegration.setRinging();
2131 }
2132 }
2133 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2134 if (isInitiator()) {
2135 this.callIntegration.setDialing();
2136 } else {
2137 this.callIntegration.setInitialized();
2138 }
2139 }
2140 case SESSION_ACCEPTED -> {
2141 if (isPeerConnectionConnected()) {
2142 this.callIntegration.setActive();
2143 } else {
2144 this.callIntegration.setInitialized();
2145 }
2146 }
2147 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2148 if (isInitiator()) {
2149 this.callIntegration.busy();
2150 } else {
2151 this.callIntegration.rejected();
2152 }
2153 }
2154 case TERMINATED_SUCCESS -> this.callIntegration.success();
2155 case ACCEPTED -> this.callIntegration.accepted();
2156 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT ->
2157 this.callIntegration.retracted();
2158 case TERMINATED_CONNECTIVITY_ERROR,
2159 TERMINATED_APPLICATION_FAILURE,
2160 TERMINATED_SECURITY_ERROR ->
2161 this.callIntegration.error();
2162 default ->
2163 throw new IllegalStateException(String.format("%s is not handled", this.state));
2164 }
2165 }
2166
2167 public ContentAddition getPendingContentAddition() {
2168 final RtpContentMap in = this.incomingContentAdd;
2169 final RtpContentMap out = this.outgoingContentAdd;
2170 if (out != null) {
2171 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2172 } else if (in != null) {
2173 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2174 } else {
2175 return null;
2176 }
2177 }
2178
2179 public Set<Media> getMedia() {
2180 final State current = getState();
2181 if (current == State.NULL) {
2182 if (isInitiator()) {
2183 return Preconditions.checkNotNull(
2184 this.proposedMedia, "RTP connection has not been initialized properly");
2185 }
2186 throw new IllegalStateException("RTP connection has not been initialized yet");
2187 }
2188 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2189 return Preconditions.checkNotNull(
2190 this.proposedMedia, "RTP connection has not been initialized properly");
2191 }
2192 final RtpContentMap localContentMap = getLocalContentMap();
2193 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2194 if (localContentMap != null) {
2195 return localContentMap.getMedia();
2196 } else if (initiatorContentMap != null) {
2197 return initiatorContentMap.getMedia();
2198 } else if (isTerminated()) {
2199 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2200 } else {
2201 return Preconditions.checkNotNull(
2202 this.proposedMedia, "RTP connection has not been initialized properly");
2203 }
2204 }
2205
2206 public boolean isVerified() {
2207 final String fingerprint = this.omemoVerification.getFingerprint();
2208 if (fingerprint == null) {
2209 return false;
2210 }
2211 final FingerprintStatus status =
2212 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2213 return status != null && status.isVerified();
2214 }
2215
2216 public boolean addMedia(final Media media) {
2217 final Set<Media> currentMedia = getMedia();
2218 if (currentMedia.contains(media)) {
2219 throw new IllegalStateException(String.format("%s has already been proposed", media));
2220 }
2221 // TODO add state protection - can only add while ACCEPTED or so
2222 Log.d(Config.LOGTAG, "adding media: " + media);
2223 return webRTCWrapper.addTrack(media);
2224 }
2225
2226 public synchronized void acceptCall() {
2227 switch (this.state) {
2228 case PROPOSED -> {
2229 cancelRingingTimeout();
2230 acceptCallFromProposed();
2231 }
2232 case SESSION_INITIALIZED -> {
2233 cancelRingingTimeout();
2234 acceptCallFromSessionInitialized();
2235 }
2236 case ACCEPTED ->
2237 Log.w(
2238 Config.LOGTAG,
2239 id.account.getJid().asBareJid()
2240 + ": the call has already been accepted with another client."
2241 + " UI was just lagging behind");
2242 case PROCEED, SESSION_ACCEPTED ->
2243 Log.w(
2244 Config.LOGTAG,
2245 id.account.getJid().asBareJid()
2246 + ": the call has already been accepted. user probably double"
2247 + " tapped the UI");
2248 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2249 }
2250 }
2251
2252 public synchronized void rejectCall() {
2253 if (isTerminated()) {
2254 Log.w(
2255 Config.LOGTAG,
2256 id.account.getJid().asBareJid()
2257 + ": received rejectCall() when session has already been terminated."
2258 + " nothing to do");
2259 return;
2260 }
2261 switch (this.state) {
2262 case PROPOSED -> rejectCallFromProposed();
2263 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2264 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2265 }
2266 }
2267
2268 public synchronized void integrationFailure() {
2269 final var state = getState();
2270 if (state == State.PROPOSED) {
2271 Log.e(
2272 Config.LOGTAG,
2273 id.account.getJid().asBareJid()
2274 + ": failed call integration in state proposed");
2275 rejectCallFromProposed();
2276 } else if (state == State.SESSION_INITIALIZED) {
2277 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2278 this.webRTCWrapper.close();
2279 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2280 } else {
2281 throw new IllegalStateException(
2282 String.format("Can not fail integration in state %s", state));
2283 }
2284 }
2285
2286 public synchronized void endCall() {
2287 if (isTerminated()) {
2288 Log.w(
2289 Config.LOGTAG,
2290 id.account.getJid().asBareJid()
2291 + ": received endCall() when session has already been terminated."
2292 + " nothing to do");
2293 return;
2294 }
2295 if (isInState(State.PROPOSED) && isResponder()) {
2296 rejectCallFromProposed();
2297 return;
2298 }
2299 if (isInState(State.PROCEED)) {
2300 if (isInitiator()) {
2301 retractFromProceed();
2302 } else {
2303 rejectCallFromProceed();
2304 }
2305 return;
2306 }
2307 if (isInitiator()
2308 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2309 this.webRTCWrapper.close();
2310 sendSessionTerminate(Reason.CANCEL);
2311 return;
2312 }
2313 if (isInState(State.SESSION_INITIALIZED)) {
2314 rejectCallFromSessionInitiate();
2315 return;
2316 }
2317 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2318 this.webRTCWrapper.close();
2319 sendSessionTerminate(Reason.SUCCESS);
2320 return;
2321 }
2322 if (isInState(
2323 State.TERMINATED_APPLICATION_FAILURE,
2324 State.TERMINATED_CONNECTIVITY_ERROR,
2325 State.TERMINATED_DECLINED_OR_BUSY)) {
2326 Log.d(
2327 Config.LOGTAG,
2328 "ignoring request to end call because already in state " + this.state);
2329 return;
2330 }
2331 throw new IllegalStateException(
2332 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2333 }
2334
2335 private void retractFromProceed() {
2336 Log.d(Config.LOGTAG, "retract from proceed");
2337 this.sendJingleMessage("retract");
2338 closeTransitionLogFinish(State.RETRACTED_RACED);
2339 }
2340
2341 private void closeTransitionLogFinish(final State state) {
2342 this.webRTCWrapper.close();
2343 transitionOrThrow(state);
2344 writeLogMessage(state);
2345 finish();
2346 }
2347
2348 private void setupWebRTC(
2349 final Set<Media> media,
2350 final Collection<PeerConnection.IceServer> iceServers,
2351 final boolean trickle)
2352 throws WebRTCWrapper.InitializationException {
2353 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2354 this.webRTCWrapper.setup(this.xmppConnectionService);
2355 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2356 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2357 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2358 }
2359
2360 private void acceptCallFromProposed() {
2361 transitionOrThrow(State.PROCEED);
2362 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2363 this.callIntegration.startAudioRouting();
2364 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2365 this.sendJingleMessage("proceed");
2366 }
2367
2368 private void rejectCallFromProposed() {
2369 transitionOrThrow(State.REJECTED);
2370 writeLogMessageMissed();
2371 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2372 this.sendJingleMessage("reject");
2373 finish();
2374 }
2375
2376 private void rejectCallFromProceed() {
2377 this.sendJingleMessage("reject");
2378 closeTransitionLogFinish(State.REJECTED_RACED);
2379 }
2380
2381 private void rejectCallFromSessionInitiate() {
2382 webRTCWrapper.close();
2383 sendSessionTerminate(Reason.DECLINE);
2384 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2385 }
2386
2387 private void sendJingleMessage(final String action) {
2388 sendJingleMessage(action, id.with);
2389 }
2390
2391 private void sendJingleMessage(final String action, final Jid to) {
2392 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2393 messagePacket.setType(
2394 im.conversations.android.xmpp.model.stanza.Message.Type
2395 .CHAT); // we want to carbon copy those
2396 messagePacket.setTo(to);
2397 final Element intent =
2398 messagePacket
2399 .addChild(action, Namespace.JINGLE_MESSAGE)
2400 .setAttribute("id", id.sessionId);
2401 if ("proceed".equals(action)) {
2402 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2403 if (isOmemoEnabled()) {
2404 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2405 final Element device =
2406 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2407 device.setAttribute("id", deviceId);
2408 }
2409 }
2410 messagePacket.addChild("store", "urn:xmpp:hints");
2411 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2412 }
2413
2414 private void sendJingleMessageFinish(final Reason reason) {
2415 final var account = id.getAccount();
2416 final var messagePacket =
2417 xmppConnectionService
2418 .getMessageGenerator()
2419 .sessionFinish(id.with, id.sessionId, reason);
2420 xmppConnectionService.sendMessagePacket(account, messagePacket);
2421 }
2422
2423 private boolean isOmemoEnabled() {
2424 final Conversational conversational = message.getConversation();
2425 if (conversational instanceof Conversation) {
2426 return ((Conversation) conversational).getNextEncryption()
2427 == Message.ENCRYPTION_AXOLOTL;
2428 }
2429 return false;
2430 }
2431
2432 private void acceptCallFromSessionInitialized() {
2433 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2434 this.callIntegration.startAudioRouting();
2435 sendSessionAccept();
2436 }
2437
2438 @Override
2439 protected synchronized boolean transition(final State target, final Runnable runnable) {
2440 if (super.transition(target, runnable)) {
2441 updateEndUserState();
2442 updateOngoingCallNotification();
2443 return true;
2444 } else {
2445 return false;
2446 }
2447 }
2448
2449 @Override
2450 public void onIceCandidate(final IceCandidate iceCandidate) {
2451 final RtpContentMap rtpContentMap =
2452 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2453 final IceUdpTransportInfo.Credentials credentials;
2454 try {
2455 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2456 } catch (final IllegalArgumentException e) {
2457 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2458 return;
2459 }
2460 final String uFrag = credentials.ufrag;
2461 final IceUdpTransportInfo.Candidate candidate =
2462 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2463 if (candidate == null) {
2464 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2465 return;
2466 }
2467 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2468 sendTransportInfo(iceCandidate.sdpMid, candidate);
2469 }
2470
2471 @Override
2472 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2473 Log.d(
2474 Config.LOGTAG,
2475 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2476 this.stateHistory.add(newState);
2477 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2478 this.sessionDuration.start();
2479 updateOngoingCallNotification();
2480 } else if (this.sessionDuration.isRunning()) {
2481 this.sessionDuration.stop();
2482 updateOngoingCallNotification();
2483 }
2484
2485 final boolean neverConnected =
2486 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2487
2488 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2489 if (neverConnected) {
2490 if (isTerminated()) {
2491 Log.d(
2492 Config.LOGTAG,
2493 id.account.getJid().asBareJid()
2494 + ": not sending session-terminate after connectivity error"
2495 + " because session is already in state "
2496 + this.state);
2497 return;
2498 }
2499 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2500 return;
2501 } else {
2502 this.restartIce();
2503 }
2504 }
2505 updateEndUserState();
2506 }
2507
2508 private void restartIce() {
2509 this.stateHistory.clear();
2510 this.webRTCWrapper.restartIceAsync();
2511 }
2512
2513 @Override
2514 public void onRenegotiationNeeded() {
2515 this.webRTCWrapper.execute(this::renegotiate);
2516 }
2517
2518 private synchronized void renegotiate() {
2519 final SessionDescription sessionDescription;
2520 try {
2521 sessionDescription = setLocalSessionDescription();
2522 } catch (final Exception e) {
2523 final Throwable cause = Throwables.getRootCause(e);
2524 webRTCWrapper.close();
2525 if (isTerminated()) {
2526 Log.d(
2527 Config.LOGTAG,
2528 "failed to renegotiate. session was already terminated",
2529 cause);
2530 return;
2531 }
2532 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2533 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2534 return;
2535 }
2536 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2537 final RtpContentMap currentContentMap = getLocalContentMap();
2538 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2539 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2540
2541 Log.d(
2542 Config.LOGTAG,
2543 id.getAccount().getJid().asBareJid()
2544 + ": renegotiate. iceRestart="
2545 + iceRestart
2546 + " content id diff="
2547 + diff);
2548
2549 if (diff.hasModifications() && iceRestart) {
2550 webRTCWrapper.close();
2551 sendSessionTerminate(
2552 Reason.FAILED_APPLICATION,
2553 "WebRTC unexpectedly tried to modify content and transport at once");
2554 return;
2555 }
2556
2557 if (iceRestart) {
2558 initiateIceRestart(rtpContentMap);
2559 return;
2560 } else if (diff.isEmpty()) {
2561 Log.d(
2562 Config.LOGTAG,
2563 "renegotiation. nothing to do. SignalingState="
2564 + this.webRTCWrapper.getSignalingState());
2565 }
2566
2567 if (diff.added.isEmpty()) {
2568 return;
2569 }
2570 modifyLocalContentMap(rtpContentMap);
2571 sendContentAdd(rtpContentMap, diff.added);
2572 }
2573
2574 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2575 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2576 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2577 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2578 iq.setTo(id.with);
2579 xmppConnectionService.sendIqPacket(
2580 id.account,
2581 iq,
2582 (response) -> {
2583 if (response.getType() == Iq.Type.RESULT) {
2584 Log.d(Config.LOGTAG, "received success to our ice restart");
2585 setLocalContentMap(rtpContentMap);
2586 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2587 return;
2588 }
2589 if (response.getType() == Iq.Type.ERROR) {
2590 if (isTieBreak(response)) {
2591 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2592 return;
2593 }
2594 handleIqErrorResponse(response);
2595 }
2596 if (response.getType() == Iq.Type.TIMEOUT) {
2597 handleIqTimeoutResponse(response);
2598 }
2599 });
2600 }
2601
2602 private boolean isTieBreak(final Iq response) {
2603 final Element error = response.findChild("error");
2604 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2605 }
2606
2607 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2608 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2609 this.outgoingContentAdd = contentAdd;
2610 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2611 prepareOutgoingContentMap(contentAdd);
2612 Futures.addCallback(
2613 outgoingContentMapFuture,
2614 new FutureCallback<>() {
2615 @Override
2616 public void onSuccess(final RtpContentMap outgoingContentMap) {
2617 sendContentAdd(outgoingContentMap);
2618 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2619 }
2620
2621 @Override
2622 public void onFailure(@NonNull Throwable throwable) {
2623 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2624 }
2625 },
2626 MoreExecutors.directExecutor());
2627 }
2628
2629 private void sendContentAdd(final RtpContentMap contentAdd) {
2630
2631 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2632 iq.setTo(id.with);
2633 xmppConnectionService.sendIqPacket(
2634 id.account,
2635 iq,
2636 (response) -> {
2637 if (response.getType() == Iq.Type.RESULT) {
2638 Log.d(
2639 Config.LOGTAG,
2640 id.getAccount().getJid().asBareJid()
2641 + ": received ACK to our content-add");
2642 return;
2643 }
2644 if (response.getType() == Iq.Type.ERROR) {
2645 if (isTieBreak(response)) {
2646 this.outgoingContentAdd = null;
2647 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2648 return;
2649 }
2650 handleIqErrorResponse(response);
2651 }
2652 if (response.getType() == Iq.Type.TIMEOUT) {
2653 handleIqTimeoutResponse(response);
2654 }
2655 });
2656 }
2657
2658 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2659 if (isInitiator()) {
2660 this.initiatorRtpContentMap = rtpContentMap;
2661 } else {
2662 this.responderRtpContentMap = rtpContentMap;
2663 }
2664 }
2665
2666 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2667 if (isInitiator()) {
2668 this.responderRtpContentMap = rtpContentMap;
2669 } else {
2670 this.initiatorRtpContentMap = rtpContentMap;
2671 }
2672 }
2673
2674 // this method is to be used for content map modifications that modify media
2675 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2676 final RtpContentMap activeContents = rtpContentMap.activeContents();
2677 setLocalContentMap(activeContents);
2678 this.callIntegration.setAudioDeviceWhenAvailable(
2679 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2680 updateEndUserState();
2681 }
2682
2683 private SessionDescription setLocalSessionDescription()
2684 throws ExecutionException, InterruptedException {
2685 final org.webrtc.SessionDescription sessionDescription =
2686 this.webRTCWrapper.setLocalDescription(false).get();
2687 return SessionDescription.parse(sessionDescription.description);
2688 }
2689
2690 private void closeWebRTCSessionAfterFailedConnection() {
2691 this.webRTCWrapper.close();
2692 synchronized (this) {
2693 if (isTerminated()) {
2694 Log.d(
2695 Config.LOGTAG,
2696 id.account.getJid().asBareJid()
2697 + ": no need to send session-terminate after failed connection."
2698 + " Other party already did");
2699 return;
2700 }
2701 setExtraState("WebRTC failed");
2702 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2703 }
2704 }
2705
2706 public boolean zeroDuration() {
2707 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2708 }
2709
2710 public long getCallDuration() {
2711 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2712 }
2713
2714 @Override
2715 public CallIntegration getCallIntegration() {
2716 return this.callIntegration;
2717 }
2718
2719 public boolean isMicrophoneEnabled() {
2720 return webRTCWrapper.isMicrophoneEnabled();
2721 }
2722
2723 public boolean setMicrophoneEnabled(final boolean enabled) {
2724 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2725 }
2726
2727 public boolean isVideoEnabled() {
2728 return webRTCWrapper.isVideoEnabled();
2729 }
2730
2731 public void setVideoEnabled(final boolean enabled) {
2732 webRTCWrapper.setVideoEnabled(enabled);
2733 }
2734
2735 public boolean isCameraSwitchable() {
2736 return webRTCWrapper.isCameraSwitchable();
2737 }
2738
2739 public boolean isFrontCamera() {
2740 return webRTCWrapper.isFrontCamera();
2741 }
2742
2743 public ListenableFuture<Boolean> switchCamera() {
2744 return webRTCWrapper.switchCamera();
2745 }
2746
2747 @Override
2748 public synchronized void onCallIntegrationShowIncomingCallUi() {
2749 if (isTerminated()) {
2750 // there might be race conditions with the call integration service invoking this
2751 // callback when the rtp session has already ended.
2752 Log.w(
2753 Config.LOGTAG,
2754 "CallIntegration requested incoming call UI but session was already"
2755 + " terminated");
2756 return;
2757 }
2758 // TODO apparently this can be called too early as well?
2759 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2760 }
2761
2762 @Override
2763 public void onCallIntegrationDisconnect() {
2764 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2765 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2766 rejectCall();
2767 } else {
2768 endCall();
2769 }
2770 }
2771
2772 @Override
2773 public void onCallIntegrationReject() {
2774 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2775 try {
2776 rejectCall();
2777 } catch (final IllegalStateException e) {
2778 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2779 }
2780 }
2781
2782 @Override
2783 public void onCallIntegrationAnswer() {
2784 // we need to start the UI to a) show it and b) be able to ask for permissions
2785 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2786 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2787 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toString());
2788 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString());
2789 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2790 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2791 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2792 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2793 xmppConnectionService.startActivity(intent);
2794 }
2795
2796 @Override
2797 public void onCallIntegrationSilence() {
2798 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2799 }
2800
2801 @Override
2802 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2803 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2804 // that was previous disabled by the user. A proper implementation would probably be to
2805 // track user choice and enable the microphone with a userEnabled() &&
2806 // callIntegration.isMicrophoneEnabled() condition
2807 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2808 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2809 }
2810
2811 @Override
2812 public void onAudioDeviceChanged(
2813 final CallIntegration.AudioDevice selectedAudioDevice,
2814 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2815 Log.d(
2816 Config.LOGTAG,
2817 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2818 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2819 selectedAudioDevice, availableAudioDevices);
2820 }
2821
2822 private void updateEndUserState() {
2823 final RtpEndUserState endUserState = getEndUserState();
2824 this.updateCallIntegrationState();
2825 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2826 id.account, id.with, id.sessionId, endUserState);
2827 }
2828
2829 private void updateOngoingCallNotification() {
2830 final State state = this.state;
2831 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2832 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2833 && isResponder()) {
2834 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2835 xmppConnectionService.removeOngoingCall();
2836 return;
2837 }
2838 final boolean reconnecting;
2839 if (state == State.SESSION_ACCEPTED) {
2840 reconnecting =
2841 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2842 } else {
2843 reconnecting = false;
2844 }
2845 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2846 } else {
2847 xmppConnectionService.removeOngoingCall();
2848 }
2849 }
2850
2851 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2852 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2853 final Iq request = new Iq(Iq.Type.GET);
2854 request.setTo(id.account.getDomain());
2855 request.addExtension(new Services());
2856 xmppConnectionService.sendIqPacket(
2857 id.account,
2858 request,
2859 (response) -> {
2860 final var iceServers = IceServers.parse(response);
2861 if (iceServers.isEmpty()) {
2862 Log.w(
2863 Config.LOGTAG,
2864 id.account.getJid().asBareJid()
2865 + ": no ICE server found "
2866 + response);
2867 }
2868 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2869 });
2870 } else {
2871 Log.w(
2872 Config.LOGTAG,
2873 id.account.getJid().asBareJid() + ": has no external service discovery");
2874 onIceServersDiscovered.onIceServersDiscovered(Collections.emptySet());
2875 }
2876 }
2877
2878 @Override
2879 protected void terminateTransport() {
2880 this.webRTCWrapper.close();
2881 }
2882
2883 @Override
2884 protected void finish() {
2885 if (isTerminated()) {
2886 this.cancelRingingTimeout();
2887 this.callIntegration.verifyDisconnected();
2888 this.webRTCWrapper.verifyClosed();
2889 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2890 super.finish();
2891 try {
2892 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2893 log.getParentFile().mkdirs();
2894 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2895 } catch (final IOException e) { }
2896 } else {
2897 throw new IllegalStateException(
2898 String.format("Unable to call finish from %s", this.state));
2899 }
2900 }
2901
2902 private void writeLogMessage(final State state) {
2903 final long duration = getCallDuration();
2904 if (state == State.TERMINATED_SUCCESS
2905 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2906 writeLogMessageSuccess(duration);
2907 } else {
2908 writeLogMessageMissed();
2909 }
2910 }
2911
2912 private void writeLogMessageSuccess(final long duration) {
2913 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2914 this.writeMessage();
2915 }
2916
2917 private void writeLogMessageMissed() {
2918 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2919 this.writeMessage();
2920 }
2921
2922 private void writeMessage() {
2923 final Conversational conversational = message.getConversation();
2924 if (conversational instanceof Conversation) {
2925 ((Conversation) conversational).add(this.message);
2926 xmppConnectionService.createMessageAsync(message);
2927 xmppConnectionService.updateConversationUi();
2928 } else {
2929 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2930 }
2931 }
2932
2933 public Optional<VideoTrack> getLocalVideoTrack() {
2934 return webRTCWrapper.getLocalVideoTrack();
2935 }
2936
2937 public Optional<VideoTrack> getRemoteVideoTrack() {
2938 return webRTCWrapper.getRemoteVideoTrack();
2939 }
2940
2941 public EglBase.Context getEglBaseContext() {
2942 return webRTCWrapper.getEglBaseContext();
2943 }
2944
2945 void setProposedMedia(final Set<Media> media) {
2946 this.proposedMedia = media;
2947 this.callIntegration.setVideoState(
2948 Media.audioOnly(media)
2949 ? VideoProfile.STATE_AUDIO_ONLY
2950 : VideoProfile.STATE_BIDIRECTIONAL);
2951 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2952 }
2953
2954 public void fireStateUpdate() {
2955 final RtpEndUserState endUserState = getEndUserState();
2956 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2957 id.account, id.with, id.sessionId, endUserState);
2958 }
2959
2960 public boolean isSwitchToVideoAvailable() {
2961 final boolean prerequisite =
2962 Media.audioOnly(getMedia())
2963 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2964 .contains(getEndUserState());
2965 return prerequisite && remoteHasVideoFeature();
2966 }
2967
2968 private boolean remoteHasVideoFeature() {
2969 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2970 }
2971
2972 private boolean remoteHasSdpOfferAnswer() {
2973 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2974 }
2975
2976 @Override
2977 public Account getAccount() {
2978 return id.account;
2979 }
2980
2981 @Override
2982 public Jid getWith() {
2983 return id.with;
2984 }
2985
2986 @Override
2987 public String getSessionId() {
2988 return id.sessionId;
2989 }
2990
2991 private interface OnIceServersDiscovered {
2992 void onIceServersDiscovered(Collection<PeerConnection.IceServer> iceServers);
2993 }
2994}