1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Account;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.services.CallIntegrationConnectionService;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import im.conversations.android.xmpp.model.disco.external.Services;
51import im.conversations.android.xmpp.model.jingle.Jingle;
52import im.conversations.android.xmpp.model.stanza.Iq;
53
54import org.webrtc.DtmfSender;
55import org.webrtc.EglBase;
56import org.webrtc.IceCandidate;
57import org.webrtc.PeerConnection;
58import org.webrtc.VideoTrack;
59
60import java.io.File;
61import java.io.IOException;
62import java.util.Arrays;
63import java.util.Collection;
64import java.util.Collections;
65import java.util.LinkedList;
66import java.util.List;
67import java.util.Map;
68import java.util.Queue;
69import java.util.Set;
70import java.util.concurrent.ExecutionException;
71import java.util.concurrent.ScheduledFuture;
72import java.util.concurrent.TimeUnit;
73import org.webrtc.EglBase;
74import org.webrtc.IceCandidate;
75import org.webrtc.PeerConnection;
76import org.webrtc.VideoTrack;
77
78public class JingleRtpConnection extends AbstractJingleConnection
79 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
80
81 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
82 public static final List<State> STATES_SHOWING_ONGOING_CALL =
83 Arrays.asList(
84 State.PROPOSED,
85 State.PROCEED,
86 State.SESSION_INITIALIZED_PRE_APPROVED,
87 State.SESSION_ACCEPTED);
88 private static final long BUSY_TIME_OUT = 30;
89
90 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
91 private final Queue<
92 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
93 pendingIceCandidates = new LinkedList<>();
94 private final OmemoVerification omemoVerification = new OmemoVerification();
95 public final CallIntegration callIntegration;
96 private final Message message;
97
98 private Set<Media> proposedMedia;
99 private RtpContentMap initiatorRtpContentMap;
100 private RtpContentMap responderRtpContentMap;
101 private RtpContentMap incomingContentAdd;
102 private RtpContentMap outgoingContentAdd;
103 private IceUdpTransportInfo.Setup peerDtlsSetup;
104 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
105 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
106 private ScheduledFuture<?> ringingTimeoutFuture;
107 private final long created = System.currentTimeMillis() / 1000L;
108
109 JingleRtpConnection(
110 final JingleConnectionManager jingleConnectionManager,
111 final Id id,
112 final Jid initiator) {
113 this(
114 jingleConnectionManager,
115 id,
116 initiator,
117 new CallIntegration(
118 jingleConnectionManager
119 .getXmppConnectionService()
120 .getApplicationContext()));
121 this.callIntegration.setAddress(
122 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
123 final var contact = id.getContact();
124 this.callIntegration.setCallerDisplayName(
125 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
126 this.callIntegration.setInitialized();
127 }
128
129 JingleRtpConnection(
130 final JingleConnectionManager jingleConnectionManager,
131 final Id id,
132 final Jid initiator,
133 final CallIntegration callIntegration) {
134 super(jingleConnectionManager, id, initiator);
135 final Conversation conversation =
136 jingleConnectionManager
137 .getXmppConnectionService()
138 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
139 this.message =
140 new Message(
141 conversation,
142 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
143 Message.TYPE_RTP_SESSION,
144 id.sessionId);
145 this.callIntegration = callIntegration;
146 this.callIntegration.setCallback(this);
147 }
148
149 @Override
150 synchronized void deliverPacket(final Iq iq) {
151 final var jingle = iq.getExtension(Jingle.class);
152 switch (jingle.getAction()) {
153 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
154 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
155 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
156 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
157 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
158 case CONTENT_ACCEPT -> receiveContentAccept(iq);
159 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
160 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
161 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
162 default -> {
163 respondOk(iq);
164 Log.d(
165 Config.LOGTAG,
166 String.format(
167 "%s: received unhandled jingle action %s",
168 id.account.getJid().asBareJid(), jingle.getAction()));
169 }
170 }
171 }
172
173 @Override
174 synchronized void notifyRebound() {
175 if (isTerminated()) {
176 return;
177 }
178 webRTCWrapper.close();
179 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
180 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
181 }
182 if (isInState(
183 State.SESSION_INITIALIZED,
184 State.SESSION_INITIALIZED_PRE_APPROVED,
185 State.SESSION_ACCEPTED)) {
186 // we might have already changed resources (full jid) at this point; so this might not
187 // even reach the other party
188 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
189 } else {
190 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
191 finish();
192 }
193 }
194
195 public boolean applyDtmfTone(final String tone) {
196 return webRTCWrapper.applyDtmfTone(tone);
197 }
198
199 private void receiveSessionTerminate(final Iq jinglePacket) {
200 respondOk(jinglePacket);
201 final var jingle = jinglePacket.getExtension(Jingle.class);
202 final Jingle.ReasonWrapper wrapper = jingle.getReason();
203 final State previous = this.state;
204 Log.d(
205 Config.LOGTAG,
206 id.account.getJid().asBareJid()
207 + ": received session terminate reason="
208 + wrapper.reason
209 + "("
210 + Strings.nullToEmpty(wrapper.text)
211 + ") while in state "
212 + previous);
213 if (TERMINATED.contains(previous)) {
214 Log.d(
215 Config.LOGTAG,
216 id.account.getJid().asBareJid()
217 + ": ignoring session terminate because already in "
218 + previous);
219 return;
220 }
221 webRTCWrapper.close();
222 final State target = reasonToState(wrapper.reason);
223 transitionOrThrow(target);
224 writeLogMessage(target);
225 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
226 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
227 }
228 finish();
229 }
230
231 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
232 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
233 // INITIALIZED only after transport-info has been received
234 if (isInState(
235 State.NULL,
236 State.PROCEED,
237 State.SESSION_INITIALIZED,
238 State.SESSION_INITIALIZED_PRE_APPROVED,
239 State.SESSION_ACCEPTED)) {
240 final RtpContentMap contentMap;
241 try {
242 contentMap = RtpContentMap.of(jingle);
243 } catch (final IllegalArgumentException | NullPointerException e) {
244 Log.d(
245 Config.LOGTAG,
246 id.account.getJid().asBareJid()
247 + ": improperly formatted contents; ignoring",
248 e);
249 respondOk(jinglePacket);
250 return;
251 }
252 receiveTransportInfo(jinglePacket, contentMap);
253 } else {
254 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
255 }
256 }
257
258 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
259 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
260 candidates = contentMap.contents.entrySet();
261 final RtpContentMap remote = getRemoteContentMap();
262 final Set<String> remoteContentIds =
263 remote == null ? Collections.emptySet() : remote.contents.keySet();
264 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
265 Log.d(
266 Config.LOGTAG,
267 "received transport-info for unknown contents "
268 + contentMap.contents.keySet()
269 + " (known: "
270 + remoteContentIds
271 + ")");
272 respondOk(jinglePacket);
273 pendingIceCandidates.addAll(candidates);
274 return;
275 }
276 if (this.state != State.SESSION_ACCEPTED) {
277 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
278 respondOk(jinglePacket);
279 pendingIceCandidates.addAll(candidates);
280 return;
281 }
282 // zero candidates + modified credentials are an ICE restart offer
283 if (checkForIceRestart(jinglePacket, contentMap)) {
284 return;
285 }
286 respondOk(jinglePacket);
287 try {
288 processCandidates(candidates);
289 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
290 Log.w(
291 Config.LOGTAG,
292 id.account.getJid().asBareJid()
293 + ": PeerConnection was not initialized when processing transport info."
294 + " this usually indicates a race condition that can be ignored");
295 }
296 }
297
298 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
299 final RtpContentMap modification;
300 try {
301 modification = RtpContentMap.of(jingle);
302 modification.requireContentDescriptions();
303 } catch (final RuntimeException e) {
304 Log.d(
305 Config.LOGTAG,
306 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
307 Throwables.getRootCause(e));
308 respondOk(iq);
309 webRTCWrapper.close();
310 sendSessionTerminate(Reason.of(e), e.getMessage());
311 return;
312 }
313 if (isInState(State.SESSION_ACCEPTED)) {
314 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
315 final ListenableFuture<RtpContentMap> future =
316 receiveRtpContentMap(
317 modification,
318 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
319 Futures.addCallback(
320 future,
321 new FutureCallback<>() {
322 @Override
323 public void onSuccess(final RtpContentMap rtpContentMap) {
324 receiveContentAdd(iq, rtpContentMap);
325 }
326
327 @Override
328 public void onFailure(@NonNull Throwable throwable) {
329 respondOk(iq);
330 final Throwable rootCause = Throwables.getRootCause(throwable);
331 Log.d(
332 Config.LOGTAG,
333 id.account.getJid().asBareJid()
334 + ": improperly formatted contents in content-add",
335 throwable);
336 webRTCWrapper.close();
337 sendSessionTerminate(
338 Reason.ofThrowable(rootCause), rootCause.getMessage());
339 }
340 },
341 MoreExecutors.directExecutor());
342 } else {
343 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
344 }
345 }
346
347 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
348 final RtpContentMap remote = getRemoteContentMap();
349 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
350 respondOk(jinglePacket);
351 this.webRTCWrapper.close();
352 sendSessionTerminate(
353 Reason.FAILED_APPLICATION,
354 String.format(
355 "contents with names %s already exists",
356 Joiner.on(", ").join(modification.getNames())));
357 return;
358 }
359 final ContentAddition contentAddition =
360 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
361
362 final RtpContentMap outgoing = this.outgoingContentAdd;
363 final Set<ContentAddition.Summary> outgoingContentAddSummary =
364 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
365
366 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
367 if (isInitiator()) {
368 Log.d(
369 Config.LOGTAG,
370 id.getAccount().getJid().asBareJid()
371 + ": respond with tie break to matching content-add offer");
372 respondWithTieBreak(jinglePacket);
373 } else {
374 Log.d(
375 Config.LOGTAG,
376 id.getAccount().getJid().asBareJid()
377 + ": automatically accept matching content-add offer");
378 acceptContentAdd(contentAddition.summary, modification);
379 }
380 return;
381 }
382
383 // once we can display multiple video tracks we can be more loose with this condition
384 // theoretically it should also be fine to automatically accept audio only contents
385 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
386 Log.d(
387 Config.LOGTAG,
388 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
389 this.incomingContentAdd = modification;
390 respondOk(jinglePacket);
391 updateEndUserState();
392 } else {
393 respondOk(jinglePacket);
394 // TODO do we want to add a reason?
395 rejectContentAdd(modification);
396 }
397 }
398
399 private void receiveContentAccept(final Iq jinglePacket) {
400 final var jingle = jinglePacket.getExtension(Jingle.class);
401 final RtpContentMap receivedContentAccept;
402 try {
403 receivedContentAccept = RtpContentMap.of(jingle);
404 receivedContentAccept.requireContentDescriptions();
405 } catch (final RuntimeException e) {
406 Log.d(
407 Config.LOGTAG,
408 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
409 Throwables.getRootCause(e));
410 respondOk(jinglePacket);
411 webRTCWrapper.close();
412 sendSessionTerminate(Reason.of(e), e.getMessage());
413 return;
414 }
415
416 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
417 if (outgoingContentAdd == null) {
418 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
419 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
420 return;
421 }
422 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
423 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
424 this.outgoingContentAdd = null;
425 respondOk(jinglePacket);
426 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
427 final ListenableFuture<RtpContentMap> future =
428 receiveRtpContentMap(
429 receivedContentAccept,
430 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
431 Futures.addCallback(
432 future,
433 new FutureCallback<>() {
434 @Override
435 public void onSuccess(final RtpContentMap result) {
436 receiveContentAccept(result);
437 }
438
439 @Override
440 public void onFailure(@NonNull final Throwable throwable) {
441 webRTCWrapper.close();
442 sendSessionTerminate(
443 Reason.ofThrowable(throwable), throwable.getMessage());
444 }
445 },
446 MoreExecutors.directExecutor());
447 } else {
448 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
449 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
450 }
451 }
452
453 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
454 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
455 final RtpContentMap modifiedContentMap =
456 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
457
458 setRemoteContentMap(modifiedContentMap);
459
460 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
461
462 final org.webrtc.SessionDescription sdp =
463 new org.webrtc.SessionDescription(
464 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
465
466 try {
467 this.webRTCWrapper.setRemoteDescription(sdp).get();
468 } catch (final Exception e) {
469 final Throwable cause = Throwables.getRootCause(e);
470 Log.d(
471 Config.LOGTAG,
472 id.getAccount().getJid().asBareJid()
473 + ": unable to set remote description after receiving content-accept",
474 cause);
475 webRTCWrapper.close();
476 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
477 return;
478 }
479 Log.d(
480 Config.LOGTAG,
481 id.getAccount().getJid().asBareJid()
482 + ": remote has accepted content-add "
483 + ContentAddition.summary(receivedContentAccept));
484 processCandidates(receivedContentAccept.contents.entrySet());
485 updateEndUserState();
486 }
487
488 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
489 if (this.state != State.SESSION_ACCEPTED) {
490 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
491 return;
492 }
493 final Map<String, Content.Senders> modification =
494 Maps.transformEntries(
495 jingle.getJingleContents(), (key, value) -> value.getSenders());
496 final boolean isInitiator = isInitiator();
497 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
498 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
499 final Set<String> currentOutgoingMediaIds =
500 currentOutgoing == null
501 ? Collections.emptySet()
502 : currentOutgoing.contents.keySet();
503 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
504 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
505 respondOk(jinglePacket);
506 final RtpContentMap modifiedContentMap;
507 try {
508 modifiedContentMap =
509 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
510 } catch (final IllegalArgumentException e) {
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
513 return;
514 }
515 this.outgoingContentAdd = modifiedContentMap;
516 Log.d(
517 Config.LOGTAG,
518 id.account.getJid().asBareJid()
519 + ": processed content-modification for pending content-add");
520 } else if (remoteContentMap != null
521 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
522 respondOk(jinglePacket);
523 final RtpContentMap modifiedRemoteContentMap;
524 try {
525 modifiedRemoteContentMap =
526 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
527 } catch (final IllegalArgumentException e) {
528 webRTCWrapper.close();
529 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
530 return;
531 }
532 final SessionDescription offer;
533 try {
534 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
535 } catch (final IllegalArgumentException | NullPointerException e) {
536 Log.d(
537 Config.LOGTAG,
538 id.getAccount().getJid().asBareJid()
539 + ": unable convert offer from content-modify to SDP",
540 e);
541 webRTCWrapper.close();
542 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
543 return;
544 }
545 Log.d(
546 Config.LOGTAG,
547 id.account.getJid().asBareJid() + ": auto accepting content-modification");
548 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
549 } else {
550 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
551 respondWithItemNotFound(jinglePacket);
552 }
553 }
554
555 private void autoAcceptContentModify(
556 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
557 this.setRemoteContentMap(modifiedRemoteContentMap);
558 final org.webrtc.SessionDescription sdp =
559 new org.webrtc.SessionDescription(
560 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
561 try {
562 this.webRTCWrapper.setRemoteDescription(sdp).get();
563 // auto accept is only done when we already have tracks
564 final SessionDescription answer = setLocalSessionDescription();
565 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
566 modifyLocalContentMap(rtpContentMap);
567 // we do not need to send an answer but do we have to resend the candidates currently in
568 // SDP?
569 // resendCandidatesFromSdp(answer);
570 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
571 } catch (final Exception e) {
572 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
573 webRTCWrapper.close();
574 sendSessionTerminate(Reason.FAILED_APPLICATION);
575 }
576 }
577
578 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
579 final SessionDescription answer) {
580 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
581 new ImmutableMultimap.Builder<>();
582 for (final SessionDescription.Media media : answer.media) {
583 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
584 if (Strings.isNullOrEmpty(mid)) {
585 continue;
586 }
587 for (final String sdpCandidate : media.attributes.get("candidate")) {
588 final IceUdpTransportInfo.Candidate candidate =
589 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
590 if (candidate != null) {
591 candidateBuilder.put(mid, candidate);
592 }
593 }
594 }
595 return candidateBuilder.build();
596 }
597
598 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
599 final RtpContentMap receivedContentReject;
600 try {
601 receivedContentReject = RtpContentMap.of(jingle);
602 } catch (final RuntimeException e) {
603 Log.d(
604 Config.LOGTAG,
605 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
606 Throwables.getRootCause(e));
607 respondOk(jinglePacket);
608 this.webRTCWrapper.close();
609 sendSessionTerminate(Reason.of(e), e.getMessage());
610 return;
611 }
612
613 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
614 if (outgoingContentAdd == null) {
615 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
616 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
617 return;
618 }
619 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
620 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
621 this.outgoingContentAdd = null;
622 respondOk(jinglePacket);
623 Log.d(Config.LOGTAG, jinglePacket.toString());
624 receiveContentReject(ourSummary);
625 } else {
626 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
627 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
628 }
629 }
630
631 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
632 try {
633 this.webRTCWrapper.removeTrack(Media.VIDEO);
634 final RtpContentMap localContentMap = customRollback();
635 modifyLocalContentMap(localContentMap);
636 } catch (final Exception e) {
637 final Throwable cause = Throwables.getRootCause(e);
638 Log.d(
639 Config.LOGTAG,
640 id.getAccount().getJid().asBareJid()
641 + ": unable to rollback local description after receiving"
642 + " content-reject",
643 cause);
644 webRTCWrapper.close();
645 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
646 return;
647 }
648 Log.d(
649 Config.LOGTAG,
650 id.getAccount().getJid().asBareJid()
651 + ": remote has rejected our content-add "
652 + summary);
653 }
654
655 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
656 final RtpContentMap receivedContentRemove;
657 try {
658 receivedContentRemove = RtpContentMap.of(jingle);
659 receivedContentRemove.requireContentDescriptions();
660 } catch (final RuntimeException e) {
661 Log.d(
662 Config.LOGTAG,
663 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
664 Throwables.getRootCause(e));
665 respondOk(jinglePacket);
666 this.webRTCWrapper.close();
667 sendSessionTerminate(Reason.of(e), e.getMessage());
668 return;
669 }
670 respondOk(jinglePacket);
671 receiveContentRemove(receivedContentRemove);
672 }
673
674 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
675 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
676 final Set<ContentAddition.Summary> contentAddSummary =
677 incomingContentAdd == null
678 ? Collections.emptySet()
679 : ContentAddition.summary(incomingContentAdd);
680 final Set<ContentAddition.Summary> removeSummary =
681 ContentAddition.summary(receivedContentRemove);
682 if (contentAddSummary.equals(removeSummary)) {
683 this.incomingContentAdd = null;
684 updateEndUserState();
685 } else {
686 webRTCWrapper.close();
687 sendSessionTerminate(
688 Reason.FAILED_APPLICATION,
689 String.format(
690 "%s only supports %s as a means to retract a not yet accepted %s",
691 BuildConfig.APP_NAME,
692 Jingle.Action.CONTENT_REMOVE,
693 Jingle.Action.CONTENT_ADD));
694 }
695 }
696
697 public synchronized void retractContentAdd() {
698 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
699 if (outgoingContentAdd == null) {
700 throw new IllegalStateException("Not outgoing content add");
701 }
702 try {
703 webRTCWrapper.removeTrack(Media.VIDEO);
704 final RtpContentMap localContentMap = customRollback();
705 modifyLocalContentMap(localContentMap);
706 } catch (final Exception e) {
707 final Throwable cause = Throwables.getRootCause(e);
708 Log.d(
709 Config.LOGTAG,
710 id.getAccount().getJid().asBareJid()
711 + ": unable to rollback local description after trying to retract"
712 + " content-add",
713 cause);
714 webRTCWrapper.close();
715 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
716 return;
717 }
718 this.outgoingContentAdd = null;
719 final Iq retract =
720 outgoingContentAdd
721 .toStub()
722 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
723 this.send(retract);
724 Log.d(
725 Config.LOGTAG,
726 id.getAccount().getJid()
727 + ": retract content-add "
728 + ContentAddition.summary(outgoingContentAdd));
729 }
730
731 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
732 final SessionDescription sdp = setLocalSessionDescription();
733 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
734 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
735 this.webRTCWrapper
736 .setRemoteDescription(
737 new org.webrtc.SessionDescription(
738 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
739 .get();
740 return localRtpContentMap;
741 }
742
743 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
744 final RtpContentMap currentRemote = getRemoteContentMap();
745 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
746 if (diff.isEmpty()) {
747 throw new IllegalStateException(
748 "Unexpected rollback condition. No difference between local and remote");
749 }
750 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
751 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
752 final RtpContentMap nextRemote =
753 currentRemote.addContent(
754 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
755 return SessionDescription.of(nextRemote, isResponder());
756 }
757 throw new IllegalStateException(
758 "Unexpected rollback condition. Senders were not uniformly none");
759 }
760
761 public synchronized void acceptContentAdd(
762 @NonNull final Set<ContentAddition.Summary> contentAddition) {
763 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
764 if (incomingContentAdd == null) {
765 throw new IllegalStateException("No incoming content add");
766 }
767
768 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
769 this.incomingContentAdd = null;
770 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
771 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
772 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
773 Log.d(
774 Config.LOGTAG,
775 "content addition is receive only. we want to upgrade to 'both'");
776 final RtpContentMap modifiedSenders =
777 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
778 final Iq proposedContentModification =
779 modifiedSenders
780 .toStub()
781 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
782 proposedContentModification.setTo(id.with);
783 xmppConnectionService.sendIqPacket(
784 id.account,
785 proposedContentModification,
786 (response) -> {
787 if (response.getType() == Iq.Type.RESULT) {
788 Log.d(
789 Config.LOGTAG,
790 id.account.getJid().asBareJid()
791 + ": remote has accepted our upgrade to"
792 + " senders=both");
793 acceptContentAdd(
794 ContentAddition.summary(modifiedSenders), modifiedSenders);
795 } else {
796 Log.d(
797 Config.LOGTAG,
798 id.account.getJid().asBareJid()
799 + ": remote has rejected our upgrade to"
800 + " senders=both");
801 acceptContentAdd(contentAddition, incomingContentAdd);
802 }
803 });
804 } else {
805 acceptContentAdd(contentAddition, incomingContentAdd);
806 }
807 } else {
808 throw new IllegalStateException(
809 "Accepted content add does not match pending content-add");
810 }
811 }
812
813 private void acceptContentAdd(
814 @NonNull final Set<ContentAddition.Summary> contentAddition,
815 final RtpContentMap incomingContentAdd) {
816 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
817 final RtpContentMap modifiedContentMap =
818 getRemoteContentMap().addContent(incomingContentAdd, setup);
819 this.setRemoteContentMap(modifiedContentMap);
820
821 final SessionDescription offer;
822 try {
823 offer = SessionDescription.of(modifiedContentMap, isResponder());
824 } catch (final IllegalArgumentException | NullPointerException e) {
825 Log.d(
826 Config.LOGTAG,
827 id.getAccount().getJid().asBareJid()
828 + ": unable convert offer from content-add to SDP",
829 e);
830 webRTCWrapper.close();
831 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
832 return;
833 }
834 this.incomingContentAdd = null;
835 acceptContentAdd(contentAddition, offer);
836 }
837
838 private void acceptContentAdd(
839 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
840 final org.webrtc.SessionDescription sdp =
841 new org.webrtc.SessionDescription(
842 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
843 try {
844 this.webRTCWrapper.setRemoteDescription(sdp).get();
845
846 // TODO add tracks for 'media' where contentAddition.senders matches
847
848 // TODO if senders.sending(isInitiator())
849
850 this.webRTCWrapper.addTrack(Media.VIDEO);
851
852 // TODO add additional transceivers for recv only cases
853
854 final SessionDescription answer = setLocalSessionDescription();
855 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
856
857 final RtpContentMap contentAcceptMap =
858 rtpContentMap.toContentModification(
859 Collections2.transform(contentAddition, ca -> ca.name));
860
861 Log.d(
862 Config.LOGTAG,
863 id.getAccount().getJid().asBareJid()
864 + ": sending content-accept "
865 + ContentAddition.summary(contentAcceptMap));
866
867 addIceCandidatesFromBlackLog();
868
869 modifyLocalContentMap(rtpContentMap);
870 final ListenableFuture<RtpContentMap> future =
871 prepareOutgoingContentMap(contentAcceptMap);
872 Futures.addCallback(
873 future,
874 new FutureCallback<>() {
875 @Override
876 public void onSuccess(final RtpContentMap rtpContentMap) {
877 sendContentAccept(rtpContentMap);
878 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
879 }
880
881 @Override
882 public void onFailure(@NonNull final Throwable throwable) {
883 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
884 }
885 },
886 MoreExecutors.directExecutor());
887 } catch (final Exception e) {
888 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
889 webRTCWrapper.close();
890 sendSessionTerminate(Reason.FAILED_APPLICATION);
891 }
892 }
893
894 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
895 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
896 send(iq);
897 }
898
899 public synchronized void rejectContentAdd() {
900 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
901 if (incomingContentAdd == null) {
902 throw new IllegalStateException("No incoming content add");
903 }
904 this.incomingContentAdd = null;
905 updateEndUserState();
906 rejectContentAdd(incomingContentAdd);
907 }
908
909 private void rejectContentAdd(final RtpContentMap contentMap) {
910 final Iq iq =
911 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
912 Log.d(
913 Config.LOGTAG,
914 id.getAccount().getJid().asBareJid()
915 + ": rejecting content "
916 + ContentAddition.summary(contentMap));
917 send(iq);
918 }
919
920 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
921 final RtpContentMap existing = getRemoteContentMap();
922 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
923 final IceUdpTransportInfo.Credentials newCredentials;
924 try {
925 existingCredentials = existing.getCredentials();
926 newCredentials = rtpContentMap.getDistinctCredentials();
927 } catch (final IllegalStateException e) {
928 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
929 return false;
930 }
931 if (existingCredentials.contains(newCredentials)) {
932 return false;
933 }
934 // TODO an alternative approach is to check if we already got an iq result to our
935 // ICE-restart
936 // and if that's the case we are seeing an answer.
937 // This might be more spec compliant but also more error prone potentially
938 final boolean isSignalStateStable =
939 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
940 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
941 final boolean isOffer = rtpContentMap.emptyCandidates();
942 final RtpContentMap restartContentMap;
943 try {
944 if (isOffer) {
945 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
946 restartContentMap =
947 existing.modifiedCredentials(
948 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
949 } else {
950 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
951 Log.d(
952 Config.LOGTAG,
953 "received confirmation of ICE restart"
954 + newCredentials
955 + " peer_setup="
956 + setup);
957 // DTLS setup attribute needs to be rewritten to reflect current peer state
958 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
959 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
960 }
961 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
962 return isOffer;
963 } else {
964 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
965 respondWithTieBreak(jinglePacket);
966 return true;
967 }
968 } catch (final Exception exception) {
969 respondOk(jinglePacket);
970 final Throwable rootCause = Throwables.getRootCause(exception);
971 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
972 // If this happens a termination is already in progress
973 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
974 return true;
975 }
976 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
977 webRTCWrapper.close();
978 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
979 return true;
980 }
981 }
982
983 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
984 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
985 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
986 throw new IllegalStateException("Invalid peer setup");
987 }
988 return peerSetup;
989 }
990
991 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
992 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
993 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
994 }
995 this.peerDtlsSetup = setup;
996 }
997
998 private boolean applyIceRestart(
999 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
1000 throws ExecutionException, InterruptedException {
1001 final SessionDescription sessionDescription =
1002 SessionDescription.of(restartContentMap, isResponder());
1003 final org.webrtc.SessionDescription.Type type =
1004 isOffer
1005 ? org.webrtc.SessionDescription.Type.OFFER
1006 : org.webrtc.SessionDescription.Type.ANSWER;
1007 org.webrtc.SessionDescription sdp =
1008 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1009 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1010 if (isInitiator()) {
1011 // We ignore the offer and respond with tie-break. This will clause the responder
1012 // not to apply the content map
1013 return false;
1014 }
1015 }
1016 webRTCWrapper.setRemoteDescription(sdp).get();
1017 setRemoteContentMap(restartContentMap);
1018 if (isOffer) {
1019 final SessionDescription localSessionDescription = setLocalSessionDescription();
1020 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1021 // We need to respond OK before sending any candidates
1022 respondOk(jinglePacket);
1023 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1024 } else {
1025 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1026 }
1027 return true;
1028 }
1029
1030 private void processCandidates(
1031 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1032 contents) {
1033 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1034 content : contents) {
1035 processCandidate(content);
1036 }
1037 }
1038
1039 private void processCandidate(
1040 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1041 content) {
1042 final RtpContentMap rtpContentMap = getRemoteContentMap();
1043 final List<String> indices = toIdentificationTags(rtpContentMap);
1044 final String sdpMid = content.getKey(); // aka content name
1045 final IceUdpTransportInfo transport = content.getValue().transport;
1046 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1047
1048 // TODO check that credentials remained the same
1049
1050 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1051 final String sdp;
1052 try {
1053 sdp = candidate.toSdpAttribute(credentials.ufrag);
1054 } catch (final IllegalArgumentException e) {
1055 Log.d(
1056 Config.LOGTAG,
1057 id.account.getJid().asBareJid()
1058 + ": ignoring invalid ICE candidate "
1059 + e.getMessage());
1060 continue;
1061 }
1062 final int mLineIndex = indices.indexOf(sdpMid);
1063 if (mLineIndex < 0) {
1064 Log.w(
1065 Config.LOGTAG,
1066 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1067 }
1068 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1069 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1070 this.webRTCWrapper.addIceCandidate(iceCandidate);
1071 }
1072 }
1073
1074 private RtpContentMap getRemoteContentMap() {
1075 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1076 }
1077
1078 private RtpContentMap getLocalContentMap() {
1079 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1080 }
1081
1082 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1083 final Group originalGroup = rtpContentMap.group;
1084 final List<String> identificationTags =
1085 originalGroup == null
1086 ? rtpContentMap.getNames()
1087 : originalGroup.getIdentificationTags();
1088 if (identificationTags.size() == 0) {
1089 Log.w(
1090 Config.LOGTAG,
1091 id.account.getJid().asBareJid()
1092 + ": no identification tags found in initial offer. we won't be able to"
1093 + " calculate mLineIndices");
1094 }
1095 return identificationTags;
1096 }
1097
1098 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1099 final Jingle jinglePacket, final boolean expectVerification) {
1100 try {
1101 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1102 } catch (final Exception e) {
1103 return Futures.immediateFailedFuture(e);
1104 }
1105 }
1106
1107 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1108 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1109 Log.d(
1110 Config.LOGTAG,
1111 "receiveRtpContentMap("
1112 + receivedContentMap.getClass().getSimpleName()
1113 + ",expectVerification="
1114 + expectVerification
1115 + ")");
1116 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1117 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1118 id.account
1119 .getAxolotlService()
1120 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1121 return Futures.transform(
1122 future,
1123 omemoVerifiedPayload -> {
1124 // TODO test if an exception here triggers a correct abort
1125 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1126 Log.d(
1127 Config.LOGTAG,
1128 id.account.getJid().asBareJid()
1129 + ": received verifiable DTLS fingerprint via "
1130 + omemoVerification);
1131 return omemoVerifiedPayload.getPayload();
1132 },
1133 MoreExecutors.directExecutor());
1134 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1135 return Futures.immediateFailedFuture(
1136 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1137 } else {
1138 return Futures.immediateFuture(receivedContentMap);
1139 }
1140 }
1141
1142 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1143 if (isInitiator()) {
1144 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1145 return;
1146 }
1147 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1148 Futures.addCallback(
1149 future,
1150 new FutureCallback<>() {
1151 @Override
1152 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1153 receiveSessionInitiate(jinglePacket, rtpContentMap);
1154 }
1155
1156 @Override
1157 public void onFailure(@NonNull final Throwable throwable) {
1158 respondOk(jinglePacket);
1159 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1160 }
1161 },
1162 MoreExecutors.directExecutor());
1163 }
1164
1165 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1166 try {
1167 contentMap.requireContentDescriptions();
1168 contentMap.requireDTLSFingerprint(true);
1169 } catch (final RuntimeException e) {
1170 Log.d(
1171 Config.LOGTAG,
1172 id.account.getJid().asBareJid() + ": improperly formatted contents",
1173 Throwables.getRootCause(e));
1174 respondOk(jinglePacket);
1175 sendSessionTerminate(Reason.of(e), e.getMessage());
1176 return;
1177 }
1178 Log.d(
1179 Config.LOGTAG,
1180 "processing session-init with " + contentMap.contents.size() + " contents");
1181 final State target;
1182 if (this.state == State.PROCEED) {
1183 Preconditions.checkState(
1184 proposedMedia != null && proposedMedia.size() > 0,
1185 "proposed media must be set when processing pre-approved session-initiate");
1186 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1187 sendSessionTerminate(
1188 Reason.SECURITY_ERROR,
1189 String.format(
1190 "Your session proposal (Jingle Message Initiation) included media"
1191 + " %s but your session-initiate was %s",
1192 this.proposedMedia, contentMap.getMedia()));
1193 return;
1194 }
1195 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1196 } else {
1197 target = State.SESSION_INITIALIZED;
1198 setProposedMedia(contentMap.getMedia());
1199 }
1200 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1201 respondOk(jinglePacket);
1202 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1203 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1204 Log.d(
1205 Config.LOGTAG,
1206 id.account.getJid().asBareJid()
1207 + ": automatically accepting session-initiate");
1208 sendSessionAccept();
1209 } else {
1210 Log.d(
1211 Config.LOGTAG,
1212 id.account.getJid().asBareJid()
1213 + ": received not pre-approved session-initiate. start ringing");
1214 startRinging();
1215 }
1216 } else {
1217 Log.d(
1218 Config.LOGTAG,
1219 String.format(
1220 "%s: received session-initiate while in state %s",
1221 id.account.getJid().asBareJid(), state));
1222 terminateWithOutOfOrder(jinglePacket);
1223 }
1224 }
1225
1226 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1227 if (isResponder()) {
1228 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1229 return;
1230 }
1231 final ListenableFuture<RtpContentMap> future =
1232 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1233 Futures.addCallback(
1234 future,
1235 new FutureCallback<>() {
1236 @Override
1237 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1238 receiveSessionAccept(jinglePacket, rtpContentMap);
1239 }
1240
1241 @Override
1242 public void onFailure(@NonNull final Throwable throwable) {
1243 respondOk(jinglePacket);
1244 Log.d(
1245 Config.LOGTAG,
1246 id.account.getJid().asBareJid()
1247 + ": improperly formatted contents in session-accept",
1248 throwable);
1249 webRTCWrapper.close();
1250 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1251 }
1252 },
1253 MoreExecutors.directExecutor());
1254 }
1255
1256 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1257 try {
1258 contentMap.requireContentDescriptions();
1259 contentMap.requireDTLSFingerprint();
1260 } catch (final RuntimeException e) {
1261 respondOk(jinglePacket);
1262 Log.d(
1263 Config.LOGTAG,
1264 id.account.getJid().asBareJid()
1265 + ": improperly formatted contents in session-accept",
1266 e);
1267 webRTCWrapper.close();
1268 sendSessionTerminate(Reason.of(e), e.getMessage());
1269 return;
1270 }
1271 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1272 if (!initiatorMedia.equals(contentMap.getMedia())) {
1273 sendSessionTerminate(
1274 Reason.SECURITY_ERROR,
1275 String.format(
1276 "Your session-included included media %s but our session-initiate was"
1277 + " %s",
1278 this.proposedMedia, contentMap.getMedia()));
1279 return;
1280 }
1281 Log.d(
1282 Config.LOGTAG,
1283 "processing session-accept with " + contentMap.contents.size() + " contents");
1284 if (transition(State.SESSION_ACCEPTED)) {
1285 respondOk(jinglePacket);
1286 receiveSessionAccept(contentMap);
1287 } else {
1288 Log.d(
1289 Config.LOGTAG,
1290 String.format(
1291 "%s: received session-accept while in state %s",
1292 id.account.getJid().asBareJid(), state));
1293 respondOk(jinglePacket);
1294 }
1295 }
1296
1297 private void receiveSessionAccept(final RtpContentMap contentMap) {
1298 this.responderRtpContentMap = contentMap;
1299 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1300 final SessionDescription sessionDescription;
1301 try {
1302 sessionDescription = SessionDescription.of(contentMap, false);
1303 } catch (final IllegalArgumentException | NullPointerException e) {
1304 Log.d(
1305 Config.LOGTAG,
1306 id.account.getJid().asBareJid()
1307 + ": unable convert offer from session-accept to SDP",
1308 e);
1309 webRTCWrapper.close();
1310 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1311 return;
1312 }
1313 final org.webrtc.SessionDescription answer =
1314 new org.webrtc.SessionDescription(
1315 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1316 try {
1317 this.webRTCWrapper.setRemoteDescription(answer).get();
1318 } catch (final Exception e) {
1319 Log.d(
1320 Config.LOGTAG,
1321 id.account.getJid().asBareJid()
1322 + ": unable to set remote description after receiving session-accept",
1323 Throwables.getRootCause(e));
1324 webRTCWrapper.close();
1325 sendSessionTerminate(
1326 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1327 return;
1328 }
1329 processCandidates(contentMap.contents.entrySet());
1330 }
1331
1332 private void sendSessionAccept() {
1333 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1334 if (rtpContentMap == null) {
1335 throw new IllegalStateException("initiator RTP Content Map has not been set");
1336 }
1337 final SessionDescription offer;
1338 try {
1339 offer = SessionDescription.of(rtpContentMap, true);
1340 } catch (final IllegalArgumentException | NullPointerException e) {
1341 Log.d(
1342 Config.LOGTAG,
1343 id.account.getJid().asBareJid()
1344 + ": unable convert offer from session-initiate to SDP",
1345 e);
1346 webRTCWrapper.close();
1347 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1348 return;
1349 }
1350 sendSessionAccept(rtpContentMap.getMedia(), offer);
1351 }
1352
1353 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1354 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1355 }
1356
1357 private synchronized void sendSessionAccept(
1358 final Set<Media> media,
1359 final SessionDescription offer,
1360 final Collection<PeerConnection.IceServer> iceServers) {
1361 if (isTerminated()) {
1362 Log.w(
1363 Config.LOGTAG,
1364 id.account.getJid().asBareJid()
1365 + ": ICE servers got discovered when session was already terminated."
1366 + " nothing to do.");
1367 return;
1368 }
1369 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1370 try {
1371 setupWebRTC(media, iceServers, !includeCandidates);
1372 } catch (final WebRTCWrapper.InitializationException e) {
1373 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1374 webRTCWrapper.close();
1375 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1376 return;
1377 }
1378 final org.webrtc.SessionDescription sdp =
1379 new org.webrtc.SessionDescription(
1380 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1381 try {
1382 this.webRTCWrapper.setRemoteDescription(sdp).get();
1383 addIceCandidatesFromBlackLog();
1384 org.webrtc.SessionDescription webRTCSessionDescription =
1385 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1386 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1387 } catch (final Exception e) {
1388 failureToAcceptSession(e);
1389 }
1390 }
1391
1392 private void failureToAcceptSession(final Throwable throwable) {
1393 if (isTerminated()) {
1394 return;
1395 }
1396 final Throwable rootCause = Throwables.getRootCause(throwable);
1397 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1398 webRTCWrapper.close();
1399 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1400 }
1401
1402 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1403 if (isTerminated()) {
1404 return;
1405 }
1406 final Throwable rootCause = Throwables.getRootCause(throwable);
1407 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1408 webRTCWrapper.close();
1409 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1410 }
1411
1412 private void addIceCandidatesFromBlackLog() {
1413 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1414 while ((foo = this.pendingIceCandidates.poll()) != null) {
1415 processCandidate(foo);
1416 Log.d(
1417 Config.LOGTAG,
1418 id.account.getJid().asBareJid() + ": added candidate from back log");
1419 }
1420 }
1421
1422 private void prepareSessionAccept(
1423 final org.webrtc.SessionDescription webRTCSessionDescription,
1424 final boolean includeCandidates) {
1425 final SessionDescription sessionDescription =
1426 SessionDescription.parse(webRTCSessionDescription.description);
1427 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1428 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1429 if (includeCandidates) {
1430 candidates = parseCandidates(sessionDescription);
1431 } else {
1432 candidates = ImmutableMultimap.of();
1433 }
1434 this.responderRtpContentMap = respondingRtpContentMap;
1435 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1436 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1437 prepareOutgoingContentMap(respondingRtpContentMap);
1438 Futures.addCallback(
1439 outgoingContentMapFuture,
1440 new FutureCallback<>() {
1441 @Override
1442 public void onSuccess(final RtpContentMap outgoingContentMap) {
1443 if (includeCandidates) {
1444 Log.d(
1445 Config.LOGTAG,
1446 "including "
1447 + candidates.size()
1448 + " candidates in session accept");
1449 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1450 } else {
1451 sendSessionAccept(outgoingContentMap);
1452 }
1453 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1454 }
1455
1456 @Override
1457 public void onFailure(@NonNull Throwable throwable) {
1458 failureToAcceptSession(throwable);
1459 }
1460 },
1461 MoreExecutors.directExecutor());
1462 }
1463
1464 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1465 if (isTerminated()) {
1466 Log.w(
1467 Config.LOGTAG,
1468 id.account.getJid().asBareJid()
1469 + ": preparing session accept was too slow. already terminated. nothing"
1470 + " to do.");
1471 return;
1472 }
1473 transitionOrThrow(State.SESSION_ACCEPTED);
1474 final Iq sessionAccept =
1475 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1476 send(sessionAccept);
1477 }
1478
1479 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1480 final RtpContentMap rtpContentMap) {
1481 if (this.omemoVerification.hasDeviceId()) {
1482 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1483 verifiedPayloadFuture =
1484 id.account
1485 .getAxolotlService()
1486 .encrypt(
1487 rtpContentMap,
1488 id.with,
1489 omemoVerification.getDeviceId());
1490 return Futures.transform(
1491 verifiedPayloadFuture,
1492 verifiedPayload -> {
1493 omemoVerification.setOrEnsureEqual(verifiedPayload);
1494 return verifiedPayload.getPayload();
1495 },
1496 MoreExecutors.directExecutor());
1497 } else {
1498 return Futures.immediateFuture(rtpContentMap);
1499 }
1500 }
1501
1502 synchronized void deliveryMessage(
1503 final Jid from,
1504 final Element message,
1505 final String serverMessageId,
1506 final long timestamp) {
1507 Log.d(
1508 Config.LOGTAG,
1509 id.account.getJid().asBareJid()
1510 + ": delivered message to JingleRtpConnection "
1511 + message);
1512 switch (message.getName()) {
1513 case "propose" ->
1514 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1515 case "proceed" ->
1516 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1517 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1518 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1519 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1520 }
1521 }
1522
1523 void deliverFailedProceed(final String message) {
1524 Log.d(
1525 Config.LOGTAG,
1526 id.account.getJid().asBareJid()
1527 + ": receive message error for proceed message ("
1528 + Strings.nullToEmpty(message)
1529 + ")");
1530 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1531 webRTCWrapper.close();
1532 Log.d(
1533 Config.LOGTAG,
1534 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1535 this.finish();
1536 }
1537 }
1538
1539 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1540 final boolean originatedFromMyself =
1541 from.asBareJid().equals(id.account.getJid().asBareJid());
1542 if (originatedFromMyself) {
1543 if (transition(State.ACCEPTED)) {
1544 acceptedOnOtherDevice(serverMsgId, timestamp);
1545 } else {
1546 Log.d(
1547 Config.LOGTAG,
1548 id.account.getJid().asBareJid()
1549 + ": unable to transition to accept because already in state="
1550 + this.state);
1551 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1552 }
1553 } else {
1554 Log.d(
1555 Config.LOGTAG,
1556 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1557 }
1558 }
1559
1560 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1561 if (serverMsgId != null) {
1562 this.message.setServerMsgId(serverMsgId);
1563 }
1564 this.message.setTime(timestamp);
1565 this.message.setCarbon(true); // indicate that call was accepted on other device
1566 this.writeLogMessageSuccess(0);
1567 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1568 this.finish();
1569 }
1570
1571 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1572 final boolean originatedFromMyself =
1573 from.asBareJid().equals(id.account.getJid().asBareJid());
1574 // reject from another one of my clients
1575 if (originatedFromMyself) {
1576 receiveRejectFromMyself(serverMsgId, timestamp);
1577 } else if (isInitiator()) {
1578 if (from.equals(id.with)) {
1579 receiveRejectFromResponder();
1580 } else {
1581 Log.d(
1582 Config.LOGTAG,
1583 id.account.getJid()
1584 + ": ignoring reject from "
1585 + from
1586 + " for session with "
1587 + id.with);
1588 }
1589 } else {
1590 Log.d(
1591 Config.LOGTAG,
1592 id.account.getJid()
1593 + ": ignoring reject from "
1594 + from
1595 + " for session with "
1596 + id.with);
1597 }
1598 }
1599
1600 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1601 if (transition(State.REJECTED)) {
1602 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1603 this.finish();
1604 if (serverMsgId != null) {
1605 this.message.setServerMsgId(serverMsgId);
1606 }
1607 this.message.setTime(timestamp);
1608 this.message.setCarbon(true); // indicate that call was rejected on other device
1609 writeLogMessageMissed();
1610 } else {
1611 Log.d(
1612 Config.LOGTAG,
1613 "not able to transition into REJECTED because already in " + this.state);
1614 }
1615 }
1616
1617 private void receiveRejectFromResponder() {
1618 if (isInState(State.PROCEED)) {
1619 Log.d(
1620 Config.LOGTAG,
1621 id.account.getJid()
1622 + ": received reject while still in proceed. callee reconsidered");
1623 closeTransitionLogFinish(State.REJECTED_RACED);
1624 return;
1625 }
1626 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1627 Log.d(
1628 Config.LOGTAG,
1629 id.account.getJid()
1630 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee"
1631 + " reconsidered before receiving session-init");
1632 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1633 return;
1634 }
1635 Log.d(
1636 Config.LOGTAG,
1637 id.account.getJid()
1638 + ": ignoring reject from responder because already in state "
1639 + this.state);
1640 }
1641
1642 private void receivePropose(
1643 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1644 final boolean originatedFromMyself =
1645 from.asBareJid().equals(id.account.getJid().asBareJid());
1646 if (originatedFromMyself) {
1647 Log.d(
1648 Config.LOGTAG,
1649 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1650 } else if (transition(
1651 State.PROPOSED,
1652 () -> {
1653 final Collection<RtpDescription> descriptions =
1654 Collections2.transform(
1655 Collections2.filter(
1656 propose.getDescriptions(),
1657 d -> d instanceof RtpDescription),
1658 input -> (RtpDescription) input);
1659 final Collection<Media> media =
1660 Collections2.transform(descriptions, RtpDescription::getMedia);
1661 Preconditions.checkState(
1662 !media.contains(Media.UNKNOWN),
1663 "RTP descriptions contain unknown media");
1664 Log.d(
1665 Config.LOGTAG,
1666 id.account.getJid().asBareJid()
1667 + ": received session proposal from "
1668 + from
1669 + " for "
1670 + media);
1671 this.setProposedMedia(Sets.newHashSet(media));
1672 })) {
1673 if (serverMsgId != null) {
1674 this.message.setServerMsgId(serverMsgId);
1675 }
1676 this.message.setTime(timestamp);
1677 startRinging();
1678 // in environments where we always use discovery timeouts we always want to respond with
1679 // 'ringing'
1680 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1681 || id.getContact().showInContactList()) {
1682 sendJingleMessage("ringing");
1683 }
1684 } else {
1685 Log.d(
1686 Config.LOGTAG,
1687 id.account.getJid()
1688 + ": ignoring session proposal because already in "
1689 + state);
1690 }
1691 }
1692
1693 private void startRinging() {
1694 this.callIntegration.setRinging();
1695 Log.d(
1696 Config.LOGTAG,
1697 id.account.getJid().asBareJid()
1698 + ": received call from "
1699 + id.with
1700 + ". start ringing");
1701 ringingTimeoutFuture =
1702 jingleConnectionManager.schedule(
1703 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1704 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1705 return;
1706 }
1707 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1708 }
1709
1710 private synchronized void ringingTimeout() {
1711 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1712 switch (this.state) {
1713 case PROPOSED -> {
1714 message.markUnread();
1715 rejectCallFromProposed();
1716 }
1717 case SESSION_INITIALIZED -> {
1718 message.markUnread();
1719 rejectCallFromSessionInitiate();
1720 }
1721 }
1722 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1723 }
1724
1725 private void cancelRingingTimeout() {
1726 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1727 if (future != null && !future.isCancelled()) {
1728 future.cancel(false);
1729 }
1730 }
1731
1732 private void receiveProceed(
1733 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1734 final Set<Media> media =
1735 Preconditions.checkNotNull(
1736 this.proposedMedia, "Proposed media has to be set before handling proceed");
1737 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1738 if (from.equals(id.with)) {
1739 if (isInitiator()) {
1740 if (transition(State.PROCEED)) {
1741 if (serverMsgId != null) {
1742 this.message.setServerMsgId(serverMsgId);
1743 }
1744 this.message.setTime(timestamp);
1745 final Integer remoteDeviceId = proceed.getDeviceId();
1746 if (isOmemoEnabled()) {
1747 this.omemoVerification.setDeviceId(remoteDeviceId);
1748 } else {
1749 if (remoteDeviceId != null) {
1750 Log.d(
1751 Config.LOGTAG,
1752 id.account.getJid().asBareJid()
1753 + ": remote party signaled support for OMEMO"
1754 + " verification but we have OMEMO disabled");
1755 }
1756 this.omemoVerification.setDeviceId(null);
1757 }
1758 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1759 } else {
1760 Log.d(
1761 Config.LOGTAG,
1762 String.format(
1763 "%s: ignoring proceed because already in %s",
1764 id.account.getJid().asBareJid(), this.state));
1765 }
1766 } else {
1767 Log.d(
1768 Config.LOGTAG,
1769 String.format(
1770 "%s: ignoring proceed because we were not initializing",
1771 id.account.getJid().asBareJid()));
1772 }
1773 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1774 if (transition(State.ACCEPTED)) {
1775 Log.d(
1776 Config.LOGTAG,
1777 id.account.getJid().asBareJid()
1778 + ": moved session with "
1779 + id.with
1780 + " into state accepted after received carbon copied proceed");
1781 acceptedOnOtherDevice(serverMsgId, timestamp);
1782 }
1783 } else {
1784 Log.d(
1785 Config.LOGTAG,
1786 String.format(
1787 "%s: ignoring proceed from %s. was expected from %s",
1788 id.account.getJid().asBareJid(), from, id.with));
1789 }
1790 }
1791
1792 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1793 if (from.equals(id.with)) {
1794 final State target =
1795 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1796 if (transition(target)) {
1797 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1798 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1799 Log.d(
1800 Config.LOGTAG,
1801 id.account.getJid().asBareJid()
1802 + ": session with "
1803 + id.with
1804 + " has been retracted (serverMsgId="
1805 + serverMsgId
1806 + ")");
1807 if (serverMsgId != null) {
1808 this.message.setServerMsgId(serverMsgId);
1809 }
1810 this.message.setTime(timestamp);
1811 if (target == State.RETRACTED) {
1812 this.message.markUnread();
1813 }
1814 writeLogMessageMissed();
1815 finish();
1816 } else {
1817 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1818 }
1819 } else {
1820 // TODO parse retract from self
1821 Log.d(
1822 Config.LOGTAG,
1823 id.account.getJid().asBareJid()
1824 + ": received retract from "
1825 + from
1826 + ". expected retract from"
1827 + id.with
1828 + ". ignoring");
1829 }
1830 }
1831
1832 public void sendSessionInitiate() {
1833 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1834 }
1835
1836 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1837 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1838 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1839 }
1840
1841 private synchronized void sendSessionInitiate(
1842 final Set<Media> media,
1843 final State targetState,
1844 final Collection<PeerConnection.IceServer> iceServers) {
1845 if (isTerminated()) {
1846 Log.w(
1847 Config.LOGTAG,
1848 id.account.getJid().asBareJid()
1849 + ": ICE servers got discovered when session was already terminated."
1850 + " nothing to do.");
1851 return;
1852 }
1853 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1854 try {
1855 setupWebRTC(media, iceServers, !includeCandidates);
1856 } catch (final WebRTCWrapper.InitializationException e) {
1857 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1858 webRTCWrapper.close();
1859 sendRetract(Reason.ofThrowable(e));
1860 return;
1861 }
1862 try {
1863 org.webrtc.SessionDescription webRTCSessionDescription =
1864 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1865 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1866 } catch (final Exception e) {
1867 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1868 // exceptions
1869 failureToInitiateSession(e, targetState);
1870 }
1871 }
1872
1873 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1874 if (isTerminated()) {
1875 return;
1876 }
1877 Log.d(
1878 Config.LOGTAG,
1879 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1880 Throwables.getRootCause(throwable));
1881 webRTCWrapper.close();
1882 final Reason reason = Reason.ofThrowable(throwable);
1883 if (isInState(targetState)) {
1884 sendSessionTerminate(reason, throwable.getMessage());
1885 } else {
1886 sendRetract(reason);
1887 }
1888 }
1889
1890 private void sendRetract(final Reason reason) {
1891 // TODO embed reason into retract
1892 sendJingleMessage("retract", id.with.asBareJid());
1893 transitionOrThrow(reasonToState(reason));
1894 this.finish();
1895 }
1896
1897 private void prepareSessionInitiate(
1898 final org.webrtc.SessionDescription webRTCSessionDescription,
1899 final boolean includeCandidates,
1900 final State targetState) {
1901 final SessionDescription sessionDescription =
1902 SessionDescription.parse(webRTCSessionDescription.description);
1903 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1904 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1905 if (includeCandidates) {
1906 candidates = parseCandidates(sessionDescription);
1907 } else {
1908 candidates = ImmutableMultimap.of();
1909 }
1910 this.initiatorRtpContentMap = rtpContentMap;
1911 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1912 encryptSessionInitiate(rtpContentMap);
1913 Futures.addCallback(
1914 outgoingContentMapFuture,
1915 new FutureCallback<>() {
1916 @Override
1917 public void onSuccess(final RtpContentMap outgoingContentMap) {
1918 if (includeCandidates) {
1919 Log.d(
1920 Config.LOGTAG,
1921 "including "
1922 + candidates.size()
1923 + " candidates in session initiate");
1924 sendSessionInitiate(
1925 outgoingContentMap.withCandidates(candidates), targetState);
1926 } else {
1927 sendSessionInitiate(outgoingContentMap, targetState);
1928 }
1929 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1930 }
1931
1932 @Override
1933 public void onFailure(@NonNull final Throwable throwable) {
1934 failureToInitiateSession(throwable, targetState);
1935 }
1936 },
1937 MoreExecutors.directExecutor());
1938 }
1939
1940 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1941 if (isTerminated()) {
1942 Log.w(
1943 Config.LOGTAG,
1944 id.account.getJid().asBareJid()
1945 + ": preparing session was too slow. already terminated. nothing to"
1946 + " do.");
1947 return;
1948 }
1949 this.transitionOrThrow(targetState);
1950 final Iq sessionInitiate =
1951 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1952 send(sessionInitiate);
1953 }
1954
1955 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1956 final RtpContentMap rtpContentMap) {
1957 if (this.omemoVerification.hasDeviceId()) {
1958 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1959 verifiedPayloadFuture =
1960 id.account
1961 .getAxolotlService()
1962 .encrypt(
1963 rtpContentMap,
1964 id.with,
1965 omemoVerification.getDeviceId());
1966 final ListenableFuture<RtpContentMap> future =
1967 Futures.transform(
1968 verifiedPayloadFuture,
1969 verifiedPayload -> {
1970 omemoVerification.setSessionFingerprint(
1971 verifiedPayload.getFingerprint());
1972 return verifiedPayload.getPayload();
1973 },
1974 MoreExecutors.directExecutor());
1975 if (Config.REQUIRE_RTP_VERIFICATION) {
1976 return future;
1977 }
1978 return Futures.catching(
1979 future,
1980 CryptoFailedException.class,
1981 e -> {
1982 Log.w(
1983 Config.LOGTAG,
1984 id.account.getJid().asBareJid()
1985 + ": unable to use OMEMO DTLS verification on outgoing"
1986 + " session initiate. falling back",
1987 e);
1988 return rtpContentMap;
1989 },
1990 MoreExecutors.directExecutor());
1991 } else {
1992 return Futures.immediateFuture(rtpContentMap);
1993 }
1994 }
1995
1996 protected void sendSessionTerminate(final Reason reason) {
1997 sendSessionTerminate(reason, null);
1998 }
1999
2000 protected void sendSessionTerminate(final Reason reason, final String text) {
2001 sendSessionTerminate(reason, text, this::writeLogMessage);
2002 sendJingleMessageFinish(reason);
2003 }
2004
2005 private void sendTransportInfo(
2006 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2007 final RtpContentMap transportInfo;
2008 try {
2009 final RtpContentMap rtpContentMap =
2010 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2011 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2012 } catch (final Exception e) {
2013 Log.d(
2014 Config.LOGTAG,
2015 id.account.getJid().asBareJid()
2016 + ": unable to prepare transport-info from candidate for content="
2017 + contentName);
2018 return;
2019 }
2020 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2021 send(iq);
2022 }
2023
2024 public RtpEndUserState getEndUserState() {
2025 switch (this.state) {
2026 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2027 if (isInitiator()) {
2028 return RtpEndUserState.RINGING;
2029 } else {
2030 return RtpEndUserState.INCOMING_CALL;
2031 }
2032 }
2033 case PROCEED -> {
2034 if (isInitiator()) {
2035 return RtpEndUserState.RINGING;
2036 } else {
2037 return RtpEndUserState.ACCEPTING_CALL;
2038 }
2039 }
2040 case SESSION_INITIALIZED_PRE_APPROVED -> {
2041 if (isInitiator()) {
2042 return RtpEndUserState.RINGING;
2043 } else {
2044 return RtpEndUserState.CONNECTING;
2045 }
2046 }
2047 case SESSION_ACCEPTED -> {
2048 final ContentAddition ca = getPendingContentAddition();
2049 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2050 return RtpEndUserState.INCOMING_CONTENT_ADD;
2051 }
2052 return getPeerConnectionStateAsEndUserState();
2053 }
2054 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2055 if (isInitiator()) {
2056 return RtpEndUserState.DECLINED_OR_BUSY;
2057 } else {
2058 return RtpEndUserState.ENDED;
2059 }
2060 }
2061 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2062 return RtpEndUserState.ENDED;
2063 }
2064 case RETRACTED_RACED -> {
2065 if (isInitiator()) {
2066 return RtpEndUserState.ENDED;
2067 } else {
2068 return RtpEndUserState.RETRACTED;
2069 }
2070 }
2071 case TERMINATED_CONNECTIVITY_ERROR -> {
2072 return zeroDuration()
2073 ? RtpEndUserState.CONNECTIVITY_ERROR
2074 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2075 }
2076 case TERMINATED_APPLICATION_FAILURE -> {
2077 return RtpEndUserState.APPLICATION_ERROR;
2078 }
2079 case TERMINATED_SECURITY_ERROR -> {
2080 return RtpEndUserState.SECURITY_ERROR;
2081 }
2082 }
2083 throw new IllegalStateException(
2084 String.format("%s has no equivalent EndUserState", this.state));
2085 }
2086
2087 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2088 final PeerConnection.PeerConnectionState state;
2089 try {
2090 state = webRTCWrapper.getState();
2091 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2092 // We usually close the WebRTCWrapper *before* transitioning so we might still
2093 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2094 return RtpEndUserState.ENDING_CALL;
2095 }
2096 return switch (state) {
2097 case CONNECTED -> RtpEndUserState.CONNECTED;
2098 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2099 case CLOSED -> RtpEndUserState.ENDING_CALL;
2100 default ->
2101 zeroDuration()
2102 ? RtpEndUserState.CONNECTIVITY_ERROR
2103 : RtpEndUserState.RECONNECTING;
2104 };
2105 }
2106
2107 private boolean isPeerConnectionConnected() {
2108 try {
2109 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2110 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2111 return false;
2112 }
2113 }
2114
2115 private void updateCallIntegrationState() {
2116 switch (this.state) {
2117 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2118 if (isInitiator()) {
2119 this.callIntegration.setDialing();
2120 } else {
2121 this.callIntegration.setRinging();
2122 }
2123 }
2124 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2125 if (isInitiator()) {
2126 this.callIntegration.setDialing();
2127 } else {
2128 this.callIntegration.setInitialized();
2129 }
2130 }
2131 case SESSION_ACCEPTED -> {
2132 if (isPeerConnectionConnected()) {
2133 this.callIntegration.setActive();
2134 } else {
2135 this.callIntegration.setInitialized();
2136 }
2137 }
2138 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2139 if (isInitiator()) {
2140 this.callIntegration.busy();
2141 } else {
2142 this.callIntegration.rejected();
2143 }
2144 }
2145 case TERMINATED_SUCCESS -> this.callIntegration.success();
2146 case ACCEPTED -> this.callIntegration.accepted();
2147 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT ->
2148 this.callIntegration.retracted();
2149 case TERMINATED_CONNECTIVITY_ERROR,
2150 TERMINATED_APPLICATION_FAILURE,
2151 TERMINATED_SECURITY_ERROR ->
2152 this.callIntegration.error();
2153 default ->
2154 throw new IllegalStateException(String.format("%s is not handled", this.state));
2155 }
2156 }
2157
2158 public ContentAddition getPendingContentAddition() {
2159 final RtpContentMap in = this.incomingContentAdd;
2160 final RtpContentMap out = this.outgoingContentAdd;
2161 if (out != null) {
2162 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2163 } else if (in != null) {
2164 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2165 } else {
2166 return null;
2167 }
2168 }
2169
2170 public Set<Media> getMedia() {
2171 final State current = getState();
2172 if (current == State.NULL) {
2173 if (isInitiator()) {
2174 return Preconditions.checkNotNull(
2175 this.proposedMedia, "RTP connection has not been initialized properly");
2176 }
2177 throw new IllegalStateException("RTP connection has not been initialized yet");
2178 }
2179 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2180 return Preconditions.checkNotNull(
2181 this.proposedMedia, "RTP connection has not been initialized properly");
2182 }
2183 final RtpContentMap localContentMap = getLocalContentMap();
2184 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2185 if (localContentMap != null) {
2186 return localContentMap.getMedia();
2187 } else if (initiatorContentMap != null) {
2188 return initiatorContentMap.getMedia();
2189 } else if (isTerminated()) {
2190 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2191 } else {
2192 return Preconditions.checkNotNull(
2193 this.proposedMedia, "RTP connection has not been initialized properly");
2194 }
2195 }
2196
2197 public boolean isVerified() {
2198 final String fingerprint = this.omemoVerification.getFingerprint();
2199 if (fingerprint == null) {
2200 return false;
2201 }
2202 final FingerprintStatus status =
2203 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2204 return status != null && status.isVerified();
2205 }
2206
2207 public boolean addMedia(final Media media) {
2208 final Set<Media> currentMedia = getMedia();
2209 if (currentMedia.contains(media)) {
2210 throw new IllegalStateException(String.format("%s has already been proposed", media));
2211 }
2212 // TODO add state protection - can only add while ACCEPTED or so
2213 Log.d(Config.LOGTAG, "adding media: " + media);
2214 return webRTCWrapper.addTrack(media);
2215 }
2216
2217 public synchronized void acceptCall() {
2218 switch (this.state) {
2219 case PROPOSED -> {
2220 cancelRingingTimeout();
2221 acceptCallFromProposed();
2222 }
2223 case SESSION_INITIALIZED -> {
2224 cancelRingingTimeout();
2225 acceptCallFromSessionInitialized();
2226 }
2227 case ACCEPTED ->
2228 Log.w(
2229 Config.LOGTAG,
2230 id.account.getJid().asBareJid()
2231 + ": the call has already been accepted with another client."
2232 + " UI was just lagging behind");
2233 case PROCEED, SESSION_ACCEPTED ->
2234 Log.w(
2235 Config.LOGTAG,
2236 id.account.getJid().asBareJid()
2237 + ": the call has already been accepted. user probably double"
2238 + " tapped the UI");
2239 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2240 }
2241 }
2242
2243 public synchronized void rejectCall() {
2244 if (isTerminated()) {
2245 Log.w(
2246 Config.LOGTAG,
2247 id.account.getJid().asBareJid()
2248 + ": received rejectCall() when session has already been terminated."
2249 + " nothing to do");
2250 return;
2251 }
2252 switch (this.state) {
2253 case PROPOSED -> rejectCallFromProposed();
2254 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2255 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2256 }
2257 }
2258
2259 public synchronized void integrationFailure() {
2260 final var state = getState();
2261 if (state == State.PROPOSED) {
2262 Log.e(
2263 Config.LOGTAG,
2264 id.account.getJid().asBareJid()
2265 + ": failed call integration in state proposed");
2266 rejectCallFromProposed();
2267 } else if (state == State.SESSION_INITIALIZED) {
2268 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2269 this.webRTCWrapper.close();
2270 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2271 } else {
2272 throw new IllegalStateException(
2273 String.format("Can not fail integration in state %s", state));
2274 }
2275 }
2276
2277 public synchronized void endCall() {
2278 if (isTerminated()) {
2279 Log.w(
2280 Config.LOGTAG,
2281 id.account.getJid().asBareJid()
2282 + ": received endCall() when session has already been terminated."
2283 + " nothing to do");
2284 return;
2285 }
2286 if (isInState(State.PROPOSED) && isResponder()) {
2287 rejectCallFromProposed();
2288 return;
2289 }
2290 if (isInState(State.PROCEED)) {
2291 if (isInitiator()) {
2292 retractFromProceed();
2293 } else {
2294 rejectCallFromProceed();
2295 }
2296 return;
2297 }
2298 if (isInitiator()
2299 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2300 this.webRTCWrapper.close();
2301 sendSessionTerminate(Reason.CANCEL);
2302 return;
2303 }
2304 if (isInState(State.SESSION_INITIALIZED)) {
2305 rejectCallFromSessionInitiate();
2306 return;
2307 }
2308 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2309 this.webRTCWrapper.close();
2310 sendSessionTerminate(Reason.SUCCESS);
2311 return;
2312 }
2313 if (isInState(
2314 State.TERMINATED_APPLICATION_FAILURE,
2315 State.TERMINATED_CONNECTIVITY_ERROR,
2316 State.TERMINATED_DECLINED_OR_BUSY)) {
2317 Log.d(
2318 Config.LOGTAG,
2319 "ignoring request to end call because already in state " + this.state);
2320 return;
2321 }
2322 throw new IllegalStateException(
2323 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2324 }
2325
2326 private void retractFromProceed() {
2327 Log.d(Config.LOGTAG, "retract from proceed");
2328 this.sendJingleMessage("retract");
2329 closeTransitionLogFinish(State.RETRACTED_RACED);
2330 }
2331
2332 private void closeTransitionLogFinish(final State state) {
2333 this.webRTCWrapper.close();
2334 transitionOrThrow(state);
2335 writeLogMessage(state);
2336 finish();
2337 }
2338
2339 private void setupWebRTC(
2340 final Set<Media> media,
2341 final Collection<PeerConnection.IceServer> iceServers,
2342 final boolean trickle)
2343 throws WebRTCWrapper.InitializationException {
2344 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2345 this.webRTCWrapper.setup(this.xmppConnectionService);
2346 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2347 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2348 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2349 }
2350
2351 private void acceptCallFromProposed() {
2352 transitionOrThrow(State.PROCEED);
2353 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2354 this.callIntegration.startAudioRouting();
2355 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2356 this.sendJingleMessage("proceed");
2357 }
2358
2359 private void rejectCallFromProposed() {
2360 transitionOrThrow(State.REJECTED);
2361 writeLogMessageMissed();
2362 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2363 this.sendJingleMessage("reject");
2364 finish();
2365 }
2366
2367 private void rejectCallFromProceed() {
2368 this.sendJingleMessage("reject");
2369 closeTransitionLogFinish(State.REJECTED_RACED);
2370 }
2371
2372 private void rejectCallFromSessionInitiate() {
2373 webRTCWrapper.close();
2374 sendSessionTerminate(Reason.DECLINE);
2375 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2376 }
2377
2378 private void sendJingleMessage(final String action) {
2379 sendJingleMessage(action, id.with);
2380 }
2381
2382 private void sendJingleMessage(final String action, final Jid to) {
2383 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2384 messagePacket.setType(
2385 im.conversations.android.xmpp.model.stanza.Message.Type
2386 .CHAT); // we want to carbon copy those
2387 messagePacket.setTo(to);
2388 final Element intent =
2389 messagePacket
2390 .addChild(action, Namespace.JINGLE_MESSAGE)
2391 .setAttribute("id", id.sessionId);
2392 if ("proceed".equals(action)) {
2393 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2394 if (isOmemoEnabled()) {
2395 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2396 final Element device =
2397 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2398 device.setAttribute("id", deviceId);
2399 }
2400 }
2401 messagePacket.addChild("store", "urn:xmpp:hints");
2402 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2403 }
2404
2405 private void sendJingleMessageFinish(final Reason reason) {
2406 final var account = id.getAccount();
2407 final var messagePacket =
2408 xmppConnectionService
2409 .getMessageGenerator()
2410 .sessionFinish(id.with, id.sessionId, reason);
2411 xmppConnectionService.sendMessagePacket(account, messagePacket);
2412 }
2413
2414 private boolean isOmemoEnabled() {
2415 final Conversational conversational = message.getConversation();
2416 if (conversational instanceof Conversation) {
2417 return ((Conversation) conversational).getNextEncryption()
2418 == Message.ENCRYPTION_AXOLOTL;
2419 }
2420 return false;
2421 }
2422
2423 private void acceptCallFromSessionInitialized() {
2424 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2425 this.callIntegration.startAudioRouting();
2426 sendSessionAccept();
2427 }
2428
2429 @Override
2430 protected synchronized boolean transition(final State target, final Runnable runnable) {
2431 if (super.transition(target, runnable)) {
2432 updateEndUserState();
2433 updateOngoingCallNotification();
2434 return true;
2435 } else {
2436 return false;
2437 }
2438 }
2439
2440 @Override
2441 public void onIceCandidate(final IceCandidate iceCandidate) {
2442 final RtpContentMap rtpContentMap =
2443 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2444 final IceUdpTransportInfo.Credentials credentials;
2445 try {
2446 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2447 } catch (final IllegalArgumentException e) {
2448 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2449 return;
2450 }
2451 final String uFrag = credentials.ufrag;
2452 final IceUdpTransportInfo.Candidate candidate =
2453 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2454 if (candidate == null) {
2455 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2456 return;
2457 }
2458 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2459 sendTransportInfo(iceCandidate.sdpMid, candidate);
2460 }
2461
2462 @Override
2463 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2464 Log.d(
2465 Config.LOGTAG,
2466 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2467 this.stateHistory.add(newState);
2468 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2469 this.sessionDuration.start();
2470 updateOngoingCallNotification();
2471 } else if (this.sessionDuration.isRunning()) {
2472 this.sessionDuration.stop();
2473 updateOngoingCallNotification();
2474 }
2475
2476 final boolean neverConnected =
2477 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2478
2479 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2480 if (neverConnected) {
2481 if (isTerminated()) {
2482 Log.d(
2483 Config.LOGTAG,
2484 id.account.getJid().asBareJid()
2485 + ": not sending session-terminate after connectivity error"
2486 + " because session is already in state "
2487 + this.state);
2488 return;
2489 }
2490 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2491 return;
2492 } else {
2493 this.restartIce();
2494 }
2495 }
2496 updateEndUserState();
2497 }
2498
2499 private void restartIce() {
2500 this.stateHistory.clear();
2501 this.webRTCWrapper.restartIceAsync();
2502 }
2503
2504 @Override
2505 public void onRenegotiationNeeded() {
2506 this.webRTCWrapper.execute(this::renegotiate);
2507 }
2508
2509 private synchronized void renegotiate() {
2510 final SessionDescription sessionDescription;
2511 try {
2512 sessionDescription = setLocalSessionDescription();
2513 } catch (final Exception e) {
2514 final Throwable cause = Throwables.getRootCause(e);
2515 webRTCWrapper.close();
2516 if (isTerminated()) {
2517 Log.d(
2518 Config.LOGTAG,
2519 "failed to renegotiate. session was already terminated",
2520 cause);
2521 return;
2522 }
2523 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2524 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2525 return;
2526 }
2527 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2528 final RtpContentMap currentContentMap = getLocalContentMap();
2529 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2530 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2531
2532 Log.d(
2533 Config.LOGTAG,
2534 id.getAccount().getJid().asBareJid()
2535 + ": renegotiate. iceRestart="
2536 + iceRestart
2537 + " content id diff="
2538 + diff);
2539
2540 if (diff.hasModifications() && iceRestart) {
2541 webRTCWrapper.close();
2542 sendSessionTerminate(
2543 Reason.FAILED_APPLICATION,
2544 "WebRTC unexpectedly tried to modify content and transport at once");
2545 return;
2546 }
2547
2548 if (iceRestart) {
2549 initiateIceRestart(rtpContentMap);
2550 return;
2551 } else if (diff.isEmpty()) {
2552 Log.d(
2553 Config.LOGTAG,
2554 "renegotiation. nothing to do. SignalingState="
2555 + this.webRTCWrapper.getSignalingState());
2556 }
2557
2558 if (diff.added.isEmpty()) {
2559 return;
2560 }
2561 modifyLocalContentMap(rtpContentMap);
2562 sendContentAdd(rtpContentMap, diff.added);
2563 }
2564
2565 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2566 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2567 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2568 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2569 iq.setTo(id.with);
2570 xmppConnectionService.sendIqPacket(
2571 id.account,
2572 iq,
2573 (response) -> {
2574 if (response.getType() == Iq.Type.RESULT) {
2575 Log.d(Config.LOGTAG, "received success to our ice restart");
2576 setLocalContentMap(rtpContentMap);
2577 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2578 return;
2579 }
2580 if (response.getType() == Iq.Type.ERROR) {
2581 if (isTieBreak(response)) {
2582 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2583 return;
2584 }
2585 handleIqErrorResponse(response);
2586 }
2587 if (response.getType() == Iq.Type.TIMEOUT) {
2588 handleIqTimeoutResponse(response);
2589 }
2590 });
2591 }
2592
2593 private boolean isTieBreak(final Iq response) {
2594 final Element error = response.findChild("error");
2595 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2596 }
2597
2598 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2599 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2600 this.outgoingContentAdd = contentAdd;
2601 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2602 prepareOutgoingContentMap(contentAdd);
2603 Futures.addCallback(
2604 outgoingContentMapFuture,
2605 new FutureCallback<>() {
2606 @Override
2607 public void onSuccess(final RtpContentMap outgoingContentMap) {
2608 sendContentAdd(outgoingContentMap);
2609 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2610 }
2611
2612 @Override
2613 public void onFailure(@NonNull Throwable throwable) {
2614 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2615 }
2616 },
2617 MoreExecutors.directExecutor());
2618 }
2619
2620 private void sendContentAdd(final RtpContentMap contentAdd) {
2621
2622 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2623 iq.setTo(id.with);
2624 xmppConnectionService.sendIqPacket(
2625 id.account,
2626 iq,
2627 (response) -> {
2628 if (response.getType() == Iq.Type.RESULT) {
2629 Log.d(
2630 Config.LOGTAG,
2631 id.getAccount().getJid().asBareJid()
2632 + ": received ACK to our content-add");
2633 return;
2634 }
2635 if (response.getType() == Iq.Type.ERROR) {
2636 if (isTieBreak(response)) {
2637 this.outgoingContentAdd = null;
2638 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2639 return;
2640 }
2641 handleIqErrorResponse(response);
2642 }
2643 if (response.getType() == Iq.Type.TIMEOUT) {
2644 handleIqTimeoutResponse(response);
2645 }
2646 });
2647 }
2648
2649 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2650 if (isInitiator()) {
2651 this.initiatorRtpContentMap = rtpContentMap;
2652 } else {
2653 this.responderRtpContentMap = rtpContentMap;
2654 }
2655 }
2656
2657 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2658 if (isInitiator()) {
2659 this.responderRtpContentMap = rtpContentMap;
2660 } else {
2661 this.initiatorRtpContentMap = rtpContentMap;
2662 }
2663 }
2664
2665 // this method is to be used for content map modifications that modify media
2666 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2667 final RtpContentMap activeContents = rtpContentMap.activeContents();
2668 setLocalContentMap(activeContents);
2669 this.callIntegration.setAudioDeviceWhenAvailable(
2670 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2671 updateEndUserState();
2672 }
2673
2674 private SessionDescription setLocalSessionDescription()
2675 throws ExecutionException, InterruptedException {
2676 final org.webrtc.SessionDescription sessionDescription =
2677 this.webRTCWrapper.setLocalDescription(false).get();
2678 return SessionDescription.parse(sessionDescription.description);
2679 }
2680
2681 private void closeWebRTCSessionAfterFailedConnection() {
2682 this.webRTCWrapper.close();
2683 synchronized (this) {
2684 if (isTerminated()) {
2685 Log.d(
2686 Config.LOGTAG,
2687 id.account.getJid().asBareJid()
2688 + ": no need to send session-terminate after failed connection."
2689 + " Other party already did");
2690 return;
2691 }
2692 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2693 }
2694 }
2695
2696 public boolean zeroDuration() {
2697 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2698 }
2699
2700 public long getCallDuration() {
2701 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2702 }
2703
2704 @Override
2705 public CallIntegration getCallIntegration() {
2706 return this.callIntegration;
2707 }
2708
2709 public boolean isMicrophoneEnabled() {
2710 return webRTCWrapper.isMicrophoneEnabled();
2711 }
2712
2713 public boolean setMicrophoneEnabled(final boolean enabled) {
2714 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2715 }
2716
2717 public boolean isVideoEnabled() {
2718 return webRTCWrapper.isVideoEnabled();
2719 }
2720
2721 public void setVideoEnabled(final boolean enabled) {
2722 webRTCWrapper.setVideoEnabled(enabled);
2723 }
2724
2725 public boolean isCameraSwitchable() {
2726 return webRTCWrapper.isCameraSwitchable();
2727 }
2728
2729 public boolean isFrontCamera() {
2730 return webRTCWrapper.isFrontCamera();
2731 }
2732
2733 public ListenableFuture<Boolean> switchCamera() {
2734 return webRTCWrapper.switchCamera();
2735 }
2736
2737 @Override
2738 public synchronized void onCallIntegrationShowIncomingCallUi() {
2739 if (isTerminated()) {
2740 // there might be race conditions with the call integration service invoking this
2741 // callback when the rtp session has already ended.
2742 Log.w(
2743 Config.LOGTAG,
2744 "CallIntegration requested incoming call UI but session was already"
2745 + " terminated");
2746 return;
2747 }
2748 // TODO apparently this can be called too early as well?
2749 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2750 }
2751
2752 @Override
2753 public void onCallIntegrationDisconnect() {
2754 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2755 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2756 rejectCall();
2757 } else {
2758 endCall();
2759 }
2760 }
2761
2762 @Override
2763 public void onCallIntegrationReject() {
2764 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2765 try {
2766 rejectCall();
2767 } catch (final IllegalStateException e) {
2768 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2769 }
2770 }
2771
2772 @Override
2773 public void onCallIntegrationAnswer() {
2774 // we need to start the UI to a) show it and b) be able to ask for permissions
2775 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2776 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2777 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toString());
2778 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString());
2779 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2780 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2781 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2782 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2783 xmppConnectionService.startActivity(intent);
2784 }
2785
2786 @Override
2787 public void onCallIntegrationSilence() {
2788 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2789 }
2790
2791 @Override
2792 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2793 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2794 // that was previous disabled by the user. A proper implementation would probably be to
2795 // track user choice and enable the microphone with a userEnabled() &&
2796 // callIntegration.isMicrophoneEnabled() condition
2797 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2798 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2799 }
2800
2801 @Override
2802 public void onAudioDeviceChanged(
2803 final CallIntegration.AudioDevice selectedAudioDevice,
2804 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2805 Log.d(
2806 Config.LOGTAG,
2807 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2808 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2809 selectedAudioDevice, availableAudioDevices);
2810 }
2811
2812 private void updateEndUserState() {
2813 final RtpEndUserState endUserState = getEndUserState();
2814 this.updateCallIntegrationState();
2815 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2816 id.account, id.with, id.sessionId, endUserState);
2817 }
2818
2819 private void updateOngoingCallNotification() {
2820 final State state = this.state;
2821 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2822 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2823 && isResponder()) {
2824 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2825 xmppConnectionService.removeOngoingCall();
2826 return;
2827 }
2828 final boolean reconnecting;
2829 if (state == State.SESSION_ACCEPTED) {
2830 reconnecting =
2831 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2832 } else {
2833 reconnecting = false;
2834 }
2835 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2836 } else {
2837 xmppConnectionService.removeOngoingCall();
2838 }
2839 }
2840
2841 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2842 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2843 final Iq request = new Iq(Iq.Type.GET);
2844 request.setTo(id.account.getDomain());
2845 request.addExtension(new Services());
2846 xmppConnectionService.sendIqPacket(
2847 id.account,
2848 request,
2849 (response) -> {
2850 final var iceServers = IceServers.parse(response);
2851 if (iceServers.isEmpty()) {
2852 Log.w(
2853 Config.LOGTAG,
2854 id.account.getJid().asBareJid()
2855 + ": no ICE server found "
2856 + response);
2857 }
2858 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2859 });
2860 } else {
2861 Log.w(
2862 Config.LOGTAG,
2863 id.account.getJid().asBareJid() + ": has no external service discovery");
2864 onIceServersDiscovered.onIceServersDiscovered(Collections.emptySet());
2865 }
2866 }
2867
2868 @Override
2869 protected void terminateTransport() {
2870 this.webRTCWrapper.close();
2871 }
2872
2873 @Override
2874 protected void finish() {
2875 if (isTerminated()) {
2876 this.cancelRingingTimeout();
2877 this.callIntegration.verifyDisconnected();
2878 this.webRTCWrapper.verifyClosed();
2879 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2880 super.finish();
2881 try {
2882 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2883 log.getParentFile().mkdirs();
2884 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2885 } catch (final IOException e) { }
2886 } else {
2887 throw new IllegalStateException(
2888 String.format("Unable to call finish from %s", this.state));
2889 }
2890 }
2891
2892 private void writeLogMessage(final State state) {
2893 final long duration = getCallDuration();
2894 if (state == State.TERMINATED_SUCCESS
2895 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2896 writeLogMessageSuccess(duration);
2897 } else {
2898 writeLogMessageMissed();
2899 }
2900 }
2901
2902 private void writeLogMessageSuccess(final long duration) {
2903 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2904 this.writeMessage();
2905 }
2906
2907 private void writeLogMessageMissed() {
2908 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2909 this.writeMessage();
2910 }
2911
2912 private void writeMessage() {
2913 final Conversational conversational = message.getConversation();
2914 if (conversational instanceof Conversation) {
2915 ((Conversation) conversational).add(this.message);
2916 xmppConnectionService.createMessageAsync(message);
2917 xmppConnectionService.updateConversationUi();
2918 } else {
2919 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2920 }
2921 }
2922
2923 public Optional<VideoTrack> getLocalVideoTrack() {
2924 return webRTCWrapper.getLocalVideoTrack();
2925 }
2926
2927 public Optional<VideoTrack> getRemoteVideoTrack() {
2928 return webRTCWrapper.getRemoteVideoTrack();
2929 }
2930
2931 public EglBase.Context getEglBaseContext() {
2932 return webRTCWrapper.getEglBaseContext();
2933 }
2934
2935 void setProposedMedia(final Set<Media> media) {
2936 this.proposedMedia = media;
2937 this.callIntegration.setVideoState(
2938 Media.audioOnly(media)
2939 ? VideoProfile.STATE_AUDIO_ONLY
2940 : VideoProfile.STATE_BIDIRECTIONAL);
2941 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2942 }
2943
2944 public void fireStateUpdate() {
2945 final RtpEndUserState endUserState = getEndUserState();
2946 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2947 id.account, id.with, id.sessionId, endUserState);
2948 }
2949
2950 public boolean isSwitchToVideoAvailable() {
2951 final boolean prerequisite =
2952 Media.audioOnly(getMedia())
2953 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2954 .contains(getEndUserState());
2955 return prerequisite && remoteHasVideoFeature();
2956 }
2957
2958 private boolean remoteHasVideoFeature() {
2959 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2960 }
2961
2962 private boolean remoteHasSdpOfferAnswer() {
2963 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2964 }
2965
2966 @Override
2967 public Account getAccount() {
2968 return id.account;
2969 }
2970
2971 @Override
2972 public Jid getWith() {
2973 return id.with;
2974 }
2975
2976 @Override
2977 public String getSessionId() {
2978 return id.sessionId;
2979 }
2980
2981 private interface OnIceServersDiscovered {
2982 void onIceServersDiscovered(Collection<PeerConnection.IceServer> iceServers);
2983 }
2984}