1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11import com.google.common.base.Joiner;
12import com.google.common.base.Optional;
13import com.google.common.base.Preconditions;
14import com.google.common.base.Stopwatch;
15import com.google.common.base.Strings;
16import com.google.common.base.Throwables;
17import com.google.common.collect.Collections2;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Account;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.services.CallIntegrationConnectionService;
39import eu.siacs.conversations.ui.RtpSessionActivity;
40import eu.siacs.conversations.xml.Element;
41import eu.siacs.conversations.xml.Namespace;
42import eu.siacs.conversations.xmpp.Jid;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
45import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import im.conversations.android.xmpp.model.jingle.Jingle;
51import im.conversations.android.xmpp.model.stanza.Iq;
52
53import org.webrtc.DtmfSender;
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.io.File;
60import java.io.IOException;
61import java.util.Arrays;
62import java.util.Collection;
63import java.util.Collections;
64import java.util.LinkedList;
65import java.util.List;
66import java.util.Map;
67import java.util.Queue;
68import java.util.Set;
69import java.util.concurrent.ExecutionException;
70import java.util.concurrent.ScheduledFuture;
71import java.util.concurrent.TimeUnit;
72import org.webrtc.EglBase;
73import org.webrtc.IceCandidate;
74import org.webrtc.PeerConnection;
75import org.webrtc.VideoTrack;
76
77public class JingleRtpConnection extends AbstractJingleConnection
78 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
79
80 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
81 public static final List<State> STATES_SHOWING_ONGOING_CALL =
82 Arrays.asList(
83 State.PROPOSED,
84 State.PROCEED,
85 State.SESSION_INITIALIZED_PRE_APPROVED,
86 State.SESSION_ACCEPTED);
87 private static final long BUSY_TIME_OUT = 30;
88
89 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
90 private final Queue<
91 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
92 pendingIceCandidates = new LinkedList<>();
93 private final OmemoVerification omemoVerification = new OmemoVerification();
94 public final CallIntegration callIntegration;
95 private final Message message;
96
97 private Set<Media> proposedMedia;
98 private RtpContentMap initiatorRtpContentMap;
99 private RtpContentMap responderRtpContentMap;
100 private RtpContentMap incomingContentAdd;
101 private RtpContentMap outgoingContentAdd;
102 private IceUdpTransportInfo.Setup peerDtlsSetup;
103 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
104 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
105 private ScheduledFuture<?> ringingTimeoutFuture;
106 private final long created = System.currentTimeMillis() / 1000L;
107
108 JingleRtpConnection(
109 final JingleConnectionManager jingleConnectionManager,
110 final Id id,
111 final Jid initiator) {
112 this(
113 jingleConnectionManager,
114 id,
115 initiator,
116 new CallIntegration(
117 jingleConnectionManager
118 .getXmppConnectionService()
119 .getApplicationContext()));
120 this.callIntegration.setAddress(
121 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
122 final var contact = id.getContact();
123 this.callIntegration.setCallerDisplayName(
124 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
125 this.callIntegration.setInitialized();
126 }
127
128 JingleRtpConnection(
129 final JingleConnectionManager jingleConnectionManager,
130 final Id id,
131 final Jid initiator,
132 final CallIntegration callIntegration) {
133 super(jingleConnectionManager, id, initiator);
134 final Conversation conversation =
135 jingleConnectionManager
136 .getXmppConnectionService()
137 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
138 this.message =
139 new Message(
140 conversation,
141 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
142 Message.TYPE_RTP_SESSION,
143 id.sessionId);
144 this.callIntegration = callIntegration;
145 this.callIntegration.setCallback(this);
146 }
147
148 @Override
149 synchronized void deliverPacket(final Iq iq) {
150 final var jingle = iq.getExtension(Jingle.class);
151 switch (jingle.getAction()) {
152 case SESSION_INITIATE -> receiveSessionInitiate(iq, jingle);
153 case TRANSPORT_INFO -> receiveTransportInfo(iq, jingle);
154 case SESSION_ACCEPT -> receiveSessionAccept(iq, jingle);
155 case SESSION_TERMINATE -> receiveSessionTerminate(iq);
156 case CONTENT_ADD -> receiveContentAdd(iq, jingle);
157 case CONTENT_ACCEPT -> receiveContentAccept(iq);
158 case CONTENT_REJECT -> receiveContentReject(iq, jingle);
159 case CONTENT_REMOVE -> receiveContentRemove(iq, jingle);
160 case CONTENT_MODIFY -> receiveContentModify(iq, jingle);
161 default -> {
162 respondOk(iq);
163 Log.d(
164 Config.LOGTAG,
165 String.format(
166 "%s: received unhandled jingle action %s",
167 id.account.getJid().asBareJid(), jingle.getAction()));
168 }
169 }
170 }
171
172 @Override
173 synchronized void notifyRebound() {
174 if (isTerminated()) {
175 return;
176 }
177 webRTCWrapper.close();
178 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
179 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
180 }
181 if (isInState(
182 State.SESSION_INITIALIZED,
183 State.SESSION_INITIALIZED_PRE_APPROVED,
184 State.SESSION_ACCEPTED)) {
185 // we might have already changed resources (full jid) at this point; so this might not
186 // even reach the other party
187 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
188 } else {
189 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
190 finish();
191 }
192 }
193
194 public boolean applyDtmfTone(final String tone) {
195 return webRTCWrapper.applyDtmfTone(tone);
196 }
197
198 private void receiveSessionTerminate(final Iq jinglePacket) {
199 respondOk(jinglePacket);
200 final var jingle = jinglePacket.getExtension(Jingle.class);
201 final Jingle.ReasonWrapper wrapper = jingle.getReason();
202 final State previous = this.state;
203 Log.d(
204 Config.LOGTAG,
205 id.account.getJid().asBareJid()
206 + ": received session terminate reason="
207 + wrapper.reason
208 + "("
209 + Strings.nullToEmpty(wrapper.text)
210 + ") while in state "
211 + previous);
212 if (TERMINATED.contains(previous)) {
213 Log.d(
214 Config.LOGTAG,
215 id.account.getJid().asBareJid()
216 + ": ignoring session terminate because already in "
217 + previous);
218 return;
219 }
220 webRTCWrapper.close();
221 final State target = reasonToState(wrapper.reason);
222 transitionOrThrow(target);
223 writeLogMessage(target);
224 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
225 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
226 }
227 finish();
228 }
229
230 private void receiveTransportInfo(final Iq jinglePacket, final Jingle jingle) {
231 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
232 // INITIALIZED only after transport-info has been received
233 if (isInState(
234 State.NULL,
235 State.PROCEED,
236 State.SESSION_INITIALIZED,
237 State.SESSION_INITIALIZED_PRE_APPROVED,
238 State.SESSION_ACCEPTED)) {
239 final RtpContentMap contentMap;
240 try {
241 contentMap = RtpContentMap.of(jingle);
242 } catch (final IllegalArgumentException | NullPointerException e) {
243 Log.d(
244 Config.LOGTAG,
245 id.account.getJid().asBareJid()
246 + ": improperly formatted contents; ignoring",
247 e);
248 respondOk(jinglePacket);
249 return;
250 }
251 receiveTransportInfo(jinglePacket, contentMap);
252 } else {
253 receiveOutOfOrderAction(jinglePacket, Jingle.Action.TRANSPORT_INFO);
254 }
255 }
256
257 private void receiveTransportInfo(final Iq jinglePacket, final RtpContentMap contentMap) {
258 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
259 candidates = contentMap.contents.entrySet();
260 final RtpContentMap remote = getRemoteContentMap();
261 final Set<String> remoteContentIds =
262 remote == null ? Collections.emptySet() : remote.contents.keySet();
263 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
264 Log.d(
265 Config.LOGTAG,
266 "received transport-info for unknown contents "
267 + contentMap.contents.keySet()
268 + " (known: "
269 + remoteContentIds
270 + ")");
271 respondOk(jinglePacket);
272 pendingIceCandidates.addAll(candidates);
273 return;
274 }
275 if (this.state != State.SESSION_ACCEPTED) {
276 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
277 respondOk(jinglePacket);
278 pendingIceCandidates.addAll(candidates);
279 return;
280 }
281 // zero candidates + modified credentials are an ICE restart offer
282 if (checkForIceRestart(jinglePacket, contentMap)) {
283 return;
284 }
285 respondOk(jinglePacket);
286 try {
287 processCandidates(candidates);
288 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
289 Log.w(
290 Config.LOGTAG,
291 id.account.getJid().asBareJid()
292 + ": PeerConnection was not initialized when processing transport info."
293 + " this usually indicates a race condition that can be ignored");
294 }
295 }
296
297 private void receiveContentAdd(final Iq iq, final Jingle jingle) {
298 final RtpContentMap modification;
299 try {
300 modification = RtpContentMap.of(jingle);
301 modification.requireContentDescriptions();
302 } catch (final RuntimeException e) {
303 Log.d(
304 Config.LOGTAG,
305 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
306 Throwables.getRootCause(e));
307 respondOk(iq);
308 webRTCWrapper.close();
309 sendSessionTerminate(Reason.of(e), e.getMessage());
310 return;
311 }
312 if (isInState(State.SESSION_ACCEPTED)) {
313 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
314 final ListenableFuture<RtpContentMap> future =
315 receiveRtpContentMap(
316 modification,
317 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
318 Futures.addCallback(
319 future,
320 new FutureCallback<>() {
321 @Override
322 public void onSuccess(final RtpContentMap rtpContentMap) {
323 receiveContentAdd(iq, rtpContentMap);
324 }
325
326 @Override
327 public void onFailure(@NonNull Throwable throwable) {
328 respondOk(iq);
329 final Throwable rootCause = Throwables.getRootCause(throwable);
330 Log.d(
331 Config.LOGTAG,
332 id.account.getJid().asBareJid()
333 + ": improperly formatted contents in content-add",
334 throwable);
335 webRTCWrapper.close();
336 sendSessionTerminate(
337 Reason.ofThrowable(rootCause), rootCause.getMessage());
338 }
339 },
340 MoreExecutors.directExecutor());
341 } else {
342 receiveOutOfOrderAction(iq, Jingle.Action.CONTENT_ADD);
343 }
344 }
345
346 private void receiveContentAdd(final Iq jinglePacket, final RtpContentMap modification) {
347 final RtpContentMap remote = getRemoteContentMap();
348 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
349 respondOk(jinglePacket);
350 this.webRTCWrapper.close();
351 sendSessionTerminate(
352 Reason.FAILED_APPLICATION,
353 String.format(
354 "contents with names %s already exists",
355 Joiner.on(", ").join(modification.getNames())));
356 return;
357 }
358 final ContentAddition contentAddition =
359 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
360
361 final RtpContentMap outgoing = this.outgoingContentAdd;
362 final Set<ContentAddition.Summary> outgoingContentAddSummary =
363 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
364
365 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
366 if (isInitiator()) {
367 Log.d(
368 Config.LOGTAG,
369 id.getAccount().getJid().asBareJid()
370 + ": respond with tie break to matching content-add offer");
371 respondWithTieBreak(jinglePacket);
372 } else {
373 Log.d(
374 Config.LOGTAG,
375 id.getAccount().getJid().asBareJid()
376 + ": automatically accept matching content-add offer");
377 acceptContentAdd(contentAddition.summary, modification);
378 }
379 return;
380 }
381
382 // once we can display multiple video tracks we can be more loose with this condition
383 // theoretically it should also be fine to automatically accept audio only contents
384 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
385 Log.d(
386 Config.LOGTAG,
387 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
388 this.incomingContentAdd = modification;
389 respondOk(jinglePacket);
390 updateEndUserState();
391 } else {
392 respondOk(jinglePacket);
393 // TODO do we want to add a reason?
394 rejectContentAdd(modification);
395 }
396 }
397
398 private void receiveContentAccept(final Iq jinglePacket) {
399 final var jingle = jinglePacket.getExtension(Jingle.class);
400 final RtpContentMap receivedContentAccept;
401 try {
402 receivedContentAccept = RtpContentMap.of(jingle);
403 receivedContentAccept.requireContentDescriptions();
404 } catch (final RuntimeException e) {
405 Log.d(
406 Config.LOGTAG,
407 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
408 Throwables.getRootCause(e));
409 respondOk(jinglePacket);
410 webRTCWrapper.close();
411 sendSessionTerminate(Reason.of(e), e.getMessage());
412 return;
413 }
414
415 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
416 if (outgoingContentAdd == null) {
417 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
418 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
419 return;
420 }
421 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
422 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
423 this.outgoingContentAdd = null;
424 respondOk(jinglePacket);
425 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
426 final ListenableFuture<RtpContentMap> future =
427 receiveRtpContentMap(
428 receivedContentAccept,
429 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
430 Futures.addCallback(
431 future,
432 new FutureCallback<>() {
433 @Override
434 public void onSuccess(final RtpContentMap result) {
435 receiveContentAccept(result);
436 }
437
438 @Override
439 public void onFailure(@NonNull final Throwable throwable) {
440 webRTCWrapper.close();
441 sendSessionTerminate(
442 Reason.ofThrowable(throwable), throwable.getMessage());
443 }
444 },
445 MoreExecutors.directExecutor());
446 } else {
447 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
448 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_ACCEPT);
449 }
450 }
451
452 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
453 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
454 final RtpContentMap modifiedContentMap =
455 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
456
457 setRemoteContentMap(modifiedContentMap);
458
459 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
460
461 final org.webrtc.SessionDescription sdp =
462 new org.webrtc.SessionDescription(
463 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
464
465 try {
466 this.webRTCWrapper.setRemoteDescription(sdp).get();
467 } catch (final Exception e) {
468 final Throwable cause = Throwables.getRootCause(e);
469 Log.d(
470 Config.LOGTAG,
471 id.getAccount().getJid().asBareJid()
472 + ": unable to set remote description after receiving content-accept",
473 cause);
474 webRTCWrapper.close();
475 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
476 return;
477 }
478 Log.d(
479 Config.LOGTAG,
480 id.getAccount().getJid().asBareJid()
481 + ": remote has accepted content-add "
482 + ContentAddition.summary(receivedContentAccept));
483 processCandidates(receivedContentAccept.contents.entrySet());
484 updateEndUserState();
485 }
486
487 private void receiveContentModify(final Iq jinglePacket, final Jingle jingle) {
488 if (this.state != State.SESSION_ACCEPTED) {
489 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_MODIFY);
490 return;
491 }
492 final Map<String, Content.Senders> modification =
493 Maps.transformEntries(
494 jingle.getJingleContents(), (key, value) -> value.getSenders());
495 final boolean isInitiator = isInitiator();
496 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
497 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
498 final Set<String> currentOutgoingMediaIds =
499 currentOutgoing == null
500 ? Collections.emptySet()
501 : currentOutgoing.contents.keySet();
502 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
503 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
504 respondOk(jinglePacket);
505 final RtpContentMap modifiedContentMap;
506 try {
507 modifiedContentMap =
508 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
509 } catch (final IllegalArgumentException e) {
510 webRTCWrapper.close();
511 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
512 return;
513 }
514 this.outgoingContentAdd = modifiedContentMap;
515 Log.d(
516 Config.LOGTAG,
517 id.account.getJid().asBareJid()
518 + ": processed content-modification for pending content-add");
519 } else if (remoteContentMap != null
520 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
521 respondOk(jinglePacket);
522 final RtpContentMap modifiedRemoteContentMap;
523 try {
524 modifiedRemoteContentMap =
525 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
526 } catch (final IllegalArgumentException e) {
527 webRTCWrapper.close();
528 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
529 return;
530 }
531 final SessionDescription offer;
532 try {
533 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
534 } catch (final IllegalArgumentException | NullPointerException e) {
535 Log.d(
536 Config.LOGTAG,
537 id.getAccount().getJid().asBareJid()
538 + ": unable convert offer from content-modify to SDP",
539 e);
540 webRTCWrapper.close();
541 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
542 return;
543 }
544 Log.d(
545 Config.LOGTAG,
546 id.account.getJid().asBareJid() + ": auto accepting content-modification");
547 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
548 } else {
549 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
550 respondWithItemNotFound(jinglePacket);
551 }
552 }
553
554 private void autoAcceptContentModify(
555 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
556 this.setRemoteContentMap(modifiedRemoteContentMap);
557 final org.webrtc.SessionDescription sdp =
558 new org.webrtc.SessionDescription(
559 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
560 try {
561 this.webRTCWrapper.setRemoteDescription(sdp).get();
562 // auto accept is only done when we already have tracks
563 final SessionDescription answer = setLocalSessionDescription();
564 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
565 modifyLocalContentMap(rtpContentMap);
566 // we do not need to send an answer but do we have to resend the candidates currently in
567 // SDP?
568 // resendCandidatesFromSdp(answer);
569 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
570 } catch (final Exception e) {
571 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
572 webRTCWrapper.close();
573 sendSessionTerminate(Reason.FAILED_APPLICATION);
574 }
575 }
576
577 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
578 final SessionDescription answer) {
579 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
580 new ImmutableMultimap.Builder<>();
581 for (final SessionDescription.Media media : answer.media) {
582 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
583 if (Strings.isNullOrEmpty(mid)) {
584 continue;
585 }
586 for (final String sdpCandidate : media.attributes.get("candidate")) {
587 final IceUdpTransportInfo.Candidate candidate =
588 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
589 if (candidate != null) {
590 candidateBuilder.put(mid, candidate);
591 }
592 }
593 }
594 return candidateBuilder.build();
595 }
596
597 private void receiveContentReject(final Iq jinglePacket, final Jingle jingle) {
598 final RtpContentMap receivedContentReject;
599 try {
600 receivedContentReject = RtpContentMap.of(jingle);
601 } catch (final RuntimeException e) {
602 Log.d(
603 Config.LOGTAG,
604 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
605 Throwables.getRootCause(e));
606 respondOk(jinglePacket);
607 this.webRTCWrapper.close();
608 sendSessionTerminate(Reason.of(e), e.getMessage());
609 return;
610 }
611
612 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
613 if (outgoingContentAdd == null) {
614 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
615 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
616 return;
617 }
618 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
619 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
620 this.outgoingContentAdd = null;
621 respondOk(jinglePacket);
622 Log.d(Config.LOGTAG, jinglePacket.toString());
623 receiveContentReject(ourSummary);
624 } else {
625 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
626 receiveOutOfOrderAction(jinglePacket, Jingle.Action.CONTENT_REJECT);
627 }
628 }
629
630 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
631 try {
632 this.webRTCWrapper.removeTrack(Media.VIDEO);
633 final RtpContentMap localContentMap = customRollback();
634 modifyLocalContentMap(localContentMap);
635 } catch (final Exception e) {
636 final Throwable cause = Throwables.getRootCause(e);
637 Log.d(
638 Config.LOGTAG,
639 id.getAccount().getJid().asBareJid()
640 + ": unable to rollback local description after receiving"
641 + " content-reject",
642 cause);
643 webRTCWrapper.close();
644 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
645 return;
646 }
647 Log.d(
648 Config.LOGTAG,
649 id.getAccount().getJid().asBareJid()
650 + ": remote has rejected our content-add "
651 + summary);
652 }
653
654 private void receiveContentRemove(final Iq jinglePacket, final Jingle jingle) {
655 final RtpContentMap receivedContentRemove;
656 try {
657 receivedContentRemove = RtpContentMap.of(jingle);
658 receivedContentRemove.requireContentDescriptions();
659 } catch (final RuntimeException e) {
660 Log.d(
661 Config.LOGTAG,
662 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
663 Throwables.getRootCause(e));
664 respondOk(jinglePacket);
665 this.webRTCWrapper.close();
666 sendSessionTerminate(Reason.of(e), e.getMessage());
667 return;
668 }
669 respondOk(jinglePacket);
670 receiveContentRemove(receivedContentRemove);
671 }
672
673 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
674 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
675 final Set<ContentAddition.Summary> contentAddSummary =
676 incomingContentAdd == null
677 ? Collections.emptySet()
678 : ContentAddition.summary(incomingContentAdd);
679 final Set<ContentAddition.Summary> removeSummary =
680 ContentAddition.summary(receivedContentRemove);
681 if (contentAddSummary.equals(removeSummary)) {
682 this.incomingContentAdd = null;
683 updateEndUserState();
684 } else {
685 webRTCWrapper.close();
686 sendSessionTerminate(
687 Reason.FAILED_APPLICATION,
688 String.format(
689 "%s only supports %s as a means to retract a not yet accepted %s",
690 BuildConfig.APP_NAME,
691 Jingle.Action.CONTENT_REMOVE,
692 Jingle.Action.CONTENT_ADD));
693 }
694 }
695
696 public synchronized void retractContentAdd() {
697 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
698 if (outgoingContentAdd == null) {
699 throw new IllegalStateException("Not outgoing content add");
700 }
701 try {
702 webRTCWrapper.removeTrack(Media.VIDEO);
703 final RtpContentMap localContentMap = customRollback();
704 modifyLocalContentMap(localContentMap);
705 } catch (final Exception e) {
706 final Throwable cause = Throwables.getRootCause(e);
707 Log.d(
708 Config.LOGTAG,
709 id.getAccount().getJid().asBareJid()
710 + ": unable to rollback local description after trying to retract"
711 + " content-add",
712 cause);
713 webRTCWrapper.close();
714 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
715 return;
716 }
717 this.outgoingContentAdd = null;
718 final Iq retract =
719 outgoingContentAdd
720 .toStub()
721 .toJinglePacket(Jingle.Action.CONTENT_REMOVE, id.sessionId);
722 this.send(retract);
723 Log.d(
724 Config.LOGTAG,
725 id.getAccount().getJid()
726 + ": retract content-add "
727 + ContentAddition.summary(outgoingContentAdd));
728 }
729
730 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
731 final SessionDescription sdp = setLocalSessionDescription();
732 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
733 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
734 this.webRTCWrapper
735 .setRemoteDescription(
736 new org.webrtc.SessionDescription(
737 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
738 .get();
739 return localRtpContentMap;
740 }
741
742 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
743 final RtpContentMap currentRemote = getRemoteContentMap();
744 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
745 if (diff.isEmpty()) {
746 throw new IllegalStateException(
747 "Unexpected rollback condition. No difference between local and remote");
748 }
749 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
750 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
751 final RtpContentMap nextRemote =
752 currentRemote.addContent(
753 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
754 return SessionDescription.of(nextRemote, isResponder());
755 }
756 throw new IllegalStateException(
757 "Unexpected rollback condition. Senders were not uniformly none");
758 }
759
760 public synchronized void acceptContentAdd(
761 @NonNull final Set<ContentAddition.Summary> contentAddition) {
762 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
763 if (incomingContentAdd == null) {
764 throw new IllegalStateException("No incoming content add");
765 }
766
767 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
768 this.incomingContentAdd = null;
769 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
770 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
771 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
772 Log.d(
773 Config.LOGTAG,
774 "content addition is receive only. we want to upgrade to 'both'");
775 final RtpContentMap modifiedSenders =
776 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
777 final Iq proposedContentModification =
778 modifiedSenders
779 .toStub()
780 .toJinglePacket(Jingle.Action.CONTENT_MODIFY, id.sessionId);
781 proposedContentModification.setTo(id.with);
782 xmppConnectionService.sendIqPacket(
783 id.account,
784 proposedContentModification,
785 (response) -> {
786 if (response.getType() == Iq.Type.RESULT) {
787 Log.d(
788 Config.LOGTAG,
789 id.account.getJid().asBareJid()
790 + ": remote has accepted our upgrade to"
791 + " senders=both");
792 acceptContentAdd(
793 ContentAddition.summary(modifiedSenders), modifiedSenders);
794 } else {
795 Log.d(
796 Config.LOGTAG,
797 id.account.getJid().asBareJid()
798 + ": remote has rejected our upgrade to"
799 + " senders=both");
800 acceptContentAdd(contentAddition, incomingContentAdd);
801 }
802 });
803 } else {
804 acceptContentAdd(contentAddition, incomingContentAdd);
805 }
806 } else {
807 throw new IllegalStateException(
808 "Accepted content add does not match pending content-add");
809 }
810 }
811
812 private void acceptContentAdd(
813 @NonNull final Set<ContentAddition.Summary> contentAddition,
814 final RtpContentMap incomingContentAdd) {
815 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
816 final RtpContentMap modifiedContentMap =
817 getRemoteContentMap().addContent(incomingContentAdd, setup);
818 this.setRemoteContentMap(modifiedContentMap);
819
820 final SessionDescription offer;
821 try {
822 offer = SessionDescription.of(modifiedContentMap, isResponder());
823 } catch (final IllegalArgumentException | NullPointerException e) {
824 Log.d(
825 Config.LOGTAG,
826 id.getAccount().getJid().asBareJid()
827 + ": unable convert offer from content-add to SDP",
828 e);
829 webRTCWrapper.close();
830 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
831 return;
832 }
833 this.incomingContentAdd = null;
834 acceptContentAdd(contentAddition, offer);
835 }
836
837 private void acceptContentAdd(
838 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
839 final org.webrtc.SessionDescription sdp =
840 new org.webrtc.SessionDescription(
841 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
842 try {
843 this.webRTCWrapper.setRemoteDescription(sdp).get();
844
845 // TODO add tracks for 'media' where contentAddition.senders matches
846
847 // TODO if senders.sending(isInitiator())
848
849 this.webRTCWrapper.addTrack(Media.VIDEO);
850
851 // TODO add additional transceivers for recv only cases
852
853 final SessionDescription answer = setLocalSessionDescription();
854 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
855
856 final RtpContentMap contentAcceptMap =
857 rtpContentMap.toContentModification(
858 Collections2.transform(contentAddition, ca -> ca.name));
859
860 Log.d(
861 Config.LOGTAG,
862 id.getAccount().getJid().asBareJid()
863 + ": sending content-accept "
864 + ContentAddition.summary(contentAcceptMap));
865
866 addIceCandidatesFromBlackLog();
867
868 modifyLocalContentMap(rtpContentMap);
869 final ListenableFuture<RtpContentMap> future =
870 prepareOutgoingContentMap(contentAcceptMap);
871 Futures.addCallback(
872 future,
873 new FutureCallback<>() {
874 @Override
875 public void onSuccess(final RtpContentMap rtpContentMap) {
876 sendContentAccept(rtpContentMap);
877 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
878 }
879
880 @Override
881 public void onFailure(@NonNull final Throwable throwable) {
882 failureToPerformAction(Jingle.Action.CONTENT_ACCEPT, throwable);
883 }
884 },
885 MoreExecutors.directExecutor());
886 } catch (final Exception e) {
887 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
888 webRTCWrapper.close();
889 sendSessionTerminate(Reason.FAILED_APPLICATION);
890 }
891 }
892
893 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
894 final Iq iq = contentAcceptMap.toJinglePacket(Jingle.Action.CONTENT_ACCEPT, id.sessionId);
895 send(iq);
896 }
897
898 public synchronized void rejectContentAdd() {
899 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
900 if (incomingContentAdd == null) {
901 throw new IllegalStateException("No incoming content add");
902 }
903 this.incomingContentAdd = null;
904 updateEndUserState();
905 rejectContentAdd(incomingContentAdd);
906 }
907
908 private void rejectContentAdd(final RtpContentMap contentMap) {
909 final Iq iq =
910 contentMap.toStub().toJinglePacket(Jingle.Action.CONTENT_REJECT, id.sessionId);
911 Log.d(
912 Config.LOGTAG,
913 id.getAccount().getJid().asBareJid()
914 + ": rejecting content "
915 + ContentAddition.summary(contentMap));
916 send(iq);
917 }
918
919 private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) {
920 final RtpContentMap existing = getRemoteContentMap();
921 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
922 final IceUdpTransportInfo.Credentials newCredentials;
923 try {
924 existingCredentials = existing.getCredentials();
925 newCredentials = rtpContentMap.getDistinctCredentials();
926 } catch (final IllegalStateException e) {
927 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
928 return false;
929 }
930 if (existingCredentials.contains(newCredentials)) {
931 return false;
932 }
933 // TODO an alternative approach is to check if we already got an iq result to our
934 // ICE-restart
935 // and if that's the case we are seeing an answer.
936 // This might be more spec compliant but also more error prone potentially
937 final boolean isSignalStateStable =
938 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
939 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
940 final boolean isOffer = rtpContentMap.emptyCandidates();
941 final RtpContentMap restartContentMap;
942 try {
943 if (isOffer) {
944 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
945 restartContentMap =
946 existing.modifiedCredentials(
947 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
948 } else {
949 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
950 Log.d(
951 Config.LOGTAG,
952 "received confirmation of ICE restart"
953 + newCredentials
954 + " peer_setup="
955 + setup);
956 // DTLS setup attribute needs to be rewritten to reflect current peer state
957 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
958 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
959 }
960 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
961 return isOffer;
962 } else {
963 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
964 respondWithTieBreak(jinglePacket);
965 return true;
966 }
967 } catch (final Exception exception) {
968 respondOk(jinglePacket);
969 final Throwable rootCause = Throwables.getRootCause(exception);
970 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
971 // If this happens a termination is already in progress
972 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
973 return true;
974 }
975 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
976 webRTCWrapper.close();
977 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
978 return true;
979 }
980 }
981
982 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
983 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
984 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
985 throw new IllegalStateException("Invalid peer setup");
986 }
987 return peerSetup;
988 }
989
990 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
991 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
992 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
993 }
994 this.peerDtlsSetup = setup;
995 }
996
997 private boolean applyIceRestart(
998 final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer)
999 throws ExecutionException, InterruptedException {
1000 final SessionDescription sessionDescription =
1001 SessionDescription.of(restartContentMap, isResponder());
1002 final org.webrtc.SessionDescription.Type type =
1003 isOffer
1004 ? org.webrtc.SessionDescription.Type.OFFER
1005 : org.webrtc.SessionDescription.Type.ANSWER;
1006 org.webrtc.SessionDescription sdp =
1007 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1008 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1009 if (isInitiator()) {
1010 // We ignore the offer and respond with tie-break. This will clause the responder
1011 // not to apply the content map
1012 return false;
1013 }
1014 }
1015 webRTCWrapper.setRemoteDescription(sdp).get();
1016 setRemoteContentMap(restartContentMap);
1017 if (isOffer) {
1018 final SessionDescription localSessionDescription = setLocalSessionDescription();
1019 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1020 // We need to respond OK before sending any candidates
1021 respondOk(jinglePacket);
1022 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1023 } else {
1024 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1025 }
1026 return true;
1027 }
1028
1029 private void processCandidates(
1030 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1031 contents) {
1032 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1033 content : contents) {
1034 processCandidate(content);
1035 }
1036 }
1037
1038 private void processCandidate(
1039 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1040 content) {
1041 final RtpContentMap rtpContentMap = getRemoteContentMap();
1042 final List<String> indices = toIdentificationTags(rtpContentMap);
1043 final String sdpMid = content.getKey(); // aka content name
1044 final IceUdpTransportInfo transport = content.getValue().transport;
1045 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1046
1047 // TODO check that credentials remained the same
1048
1049 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1050 final String sdp;
1051 try {
1052 sdp = candidate.toSdpAttribute(credentials.ufrag);
1053 } catch (final IllegalArgumentException e) {
1054 Log.d(
1055 Config.LOGTAG,
1056 id.account.getJid().asBareJid()
1057 + ": ignoring invalid ICE candidate "
1058 + e.getMessage());
1059 continue;
1060 }
1061 final int mLineIndex = indices.indexOf(sdpMid);
1062 if (mLineIndex < 0) {
1063 Log.w(
1064 Config.LOGTAG,
1065 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1066 }
1067 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1068 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1069 this.webRTCWrapper.addIceCandidate(iceCandidate);
1070 }
1071 }
1072
1073 private RtpContentMap getRemoteContentMap() {
1074 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1075 }
1076
1077 private RtpContentMap getLocalContentMap() {
1078 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1079 }
1080
1081 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1082 final Group originalGroup = rtpContentMap.group;
1083 final List<String> identificationTags =
1084 originalGroup == null
1085 ? rtpContentMap.getNames()
1086 : originalGroup.getIdentificationTags();
1087 if (identificationTags.size() == 0) {
1088 Log.w(
1089 Config.LOGTAG,
1090 id.account.getJid().asBareJid()
1091 + ": no identification tags found in initial offer. we won't be able to"
1092 + " calculate mLineIndices");
1093 }
1094 return identificationTags;
1095 }
1096
1097 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1098 final Jingle jinglePacket, final boolean expectVerification) {
1099 try {
1100 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1101 } catch (final Exception e) {
1102 return Futures.immediateFailedFuture(e);
1103 }
1104 }
1105
1106 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1107 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1108 Log.d(
1109 Config.LOGTAG,
1110 "receiveRtpContentMap("
1111 + receivedContentMap.getClass().getSimpleName()
1112 + ",expectVerification="
1113 + expectVerification
1114 + ")");
1115 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1116 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1117 id.account
1118 .getAxolotlService()
1119 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1120 return Futures.transform(
1121 future,
1122 omemoVerifiedPayload -> {
1123 // TODO test if an exception here triggers a correct abort
1124 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1125 Log.d(
1126 Config.LOGTAG,
1127 id.account.getJid().asBareJid()
1128 + ": received verifiable DTLS fingerprint via "
1129 + omemoVerification);
1130 return omemoVerifiedPayload.getPayload();
1131 },
1132 MoreExecutors.directExecutor());
1133 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1134 return Futures.immediateFailedFuture(
1135 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1136 } else {
1137 return Futures.immediateFuture(receivedContentMap);
1138 }
1139 }
1140
1141 private void receiveSessionInitiate(final Iq jinglePacket, final Jingle jingle) {
1142 if (isInitiator()) {
1143 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_INITIATE);
1144 return;
1145 }
1146 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jingle, false);
1147 Futures.addCallback(
1148 future,
1149 new FutureCallback<>() {
1150 @Override
1151 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1152 receiveSessionInitiate(jinglePacket, rtpContentMap);
1153 }
1154
1155 @Override
1156 public void onFailure(@NonNull final Throwable throwable) {
1157 respondOk(jinglePacket);
1158 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1159 }
1160 },
1161 MoreExecutors.directExecutor());
1162 }
1163
1164 private void receiveSessionInitiate(final Iq jinglePacket, final RtpContentMap contentMap) {
1165 try {
1166 contentMap.requireContentDescriptions();
1167 contentMap.requireDTLSFingerprint(true);
1168 } catch (final RuntimeException e) {
1169 Log.d(
1170 Config.LOGTAG,
1171 id.account.getJid().asBareJid() + ": improperly formatted contents",
1172 Throwables.getRootCause(e));
1173 respondOk(jinglePacket);
1174 sendSessionTerminate(Reason.of(e), e.getMessage());
1175 return;
1176 }
1177 Log.d(
1178 Config.LOGTAG,
1179 "processing session-init with " + contentMap.contents.size() + " contents");
1180 final State target;
1181 if (this.state == State.PROCEED) {
1182 Preconditions.checkState(
1183 proposedMedia != null && proposedMedia.size() > 0,
1184 "proposed media must be set when processing pre-approved session-initiate");
1185 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1186 sendSessionTerminate(
1187 Reason.SECURITY_ERROR,
1188 String.format(
1189 "Your session proposal (Jingle Message Initiation) included media"
1190 + " %s but your session-initiate was %s",
1191 this.proposedMedia, contentMap.getMedia()));
1192 return;
1193 }
1194 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1195 } else {
1196 target = State.SESSION_INITIALIZED;
1197 setProposedMedia(contentMap.getMedia());
1198 }
1199 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1200 respondOk(jinglePacket);
1201 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1202 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1203 Log.d(
1204 Config.LOGTAG,
1205 id.account.getJid().asBareJid()
1206 + ": automatically accepting session-initiate");
1207 sendSessionAccept();
1208 } else {
1209 Log.d(
1210 Config.LOGTAG,
1211 id.account.getJid().asBareJid()
1212 + ": received not pre-approved session-initiate. start ringing");
1213 startRinging();
1214 }
1215 } else {
1216 Log.d(
1217 Config.LOGTAG,
1218 String.format(
1219 "%s: received session-initiate while in state %s",
1220 id.account.getJid().asBareJid(), state));
1221 terminateWithOutOfOrder(jinglePacket);
1222 }
1223 }
1224
1225 private void receiveSessionAccept(final Iq jinglePacket, final Jingle jingle) {
1226 if (isResponder()) {
1227 receiveOutOfOrderAction(jinglePacket, Jingle.Action.SESSION_ACCEPT);
1228 return;
1229 }
1230 final ListenableFuture<RtpContentMap> future =
1231 receiveRtpContentMap(jingle, this.omemoVerification.hasFingerprint());
1232 Futures.addCallback(
1233 future,
1234 new FutureCallback<>() {
1235 @Override
1236 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1237 receiveSessionAccept(jinglePacket, rtpContentMap);
1238 }
1239
1240 @Override
1241 public void onFailure(@NonNull final Throwable throwable) {
1242 respondOk(jinglePacket);
1243 Log.d(
1244 Config.LOGTAG,
1245 id.account.getJid().asBareJid()
1246 + ": improperly formatted contents in session-accept",
1247 throwable);
1248 webRTCWrapper.close();
1249 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1250 }
1251 },
1252 MoreExecutors.directExecutor());
1253 }
1254
1255 private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) {
1256 try {
1257 contentMap.requireContentDescriptions();
1258 contentMap.requireDTLSFingerprint();
1259 } catch (final RuntimeException e) {
1260 respondOk(jinglePacket);
1261 Log.d(
1262 Config.LOGTAG,
1263 id.account.getJid().asBareJid()
1264 + ": improperly formatted contents in session-accept",
1265 e);
1266 webRTCWrapper.close();
1267 sendSessionTerminate(Reason.of(e), e.getMessage());
1268 return;
1269 }
1270 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1271 if (!initiatorMedia.equals(contentMap.getMedia())) {
1272 sendSessionTerminate(
1273 Reason.SECURITY_ERROR,
1274 String.format(
1275 "Your session-included included media %s but our session-initiate was"
1276 + " %s",
1277 this.proposedMedia, contentMap.getMedia()));
1278 return;
1279 }
1280 Log.d(
1281 Config.LOGTAG,
1282 "processing session-accept with " + contentMap.contents.size() + " contents");
1283 if (transition(State.SESSION_ACCEPTED)) {
1284 respondOk(jinglePacket);
1285 receiveSessionAccept(contentMap);
1286 } else {
1287 Log.d(
1288 Config.LOGTAG,
1289 String.format(
1290 "%s: received session-accept while in state %s",
1291 id.account.getJid().asBareJid(), state));
1292 respondOk(jinglePacket);
1293 }
1294 }
1295
1296 private void receiveSessionAccept(final RtpContentMap contentMap) {
1297 this.responderRtpContentMap = contentMap;
1298 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1299 final SessionDescription sessionDescription;
1300 try {
1301 sessionDescription = SessionDescription.of(contentMap, false);
1302 } catch (final IllegalArgumentException | NullPointerException e) {
1303 Log.d(
1304 Config.LOGTAG,
1305 id.account.getJid().asBareJid()
1306 + ": unable convert offer from session-accept to SDP",
1307 e);
1308 webRTCWrapper.close();
1309 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1310 return;
1311 }
1312 final org.webrtc.SessionDescription answer =
1313 new org.webrtc.SessionDescription(
1314 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1315 try {
1316 this.webRTCWrapper.setRemoteDescription(answer).get();
1317 } catch (final Exception e) {
1318 Log.d(
1319 Config.LOGTAG,
1320 id.account.getJid().asBareJid()
1321 + ": unable to set remote description after receiving session-accept",
1322 Throwables.getRootCause(e));
1323 webRTCWrapper.close();
1324 sendSessionTerminate(
1325 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1326 return;
1327 }
1328 processCandidates(contentMap.contents.entrySet());
1329 }
1330
1331 private void sendSessionAccept() {
1332 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1333 if (rtpContentMap == null) {
1334 throw new IllegalStateException("initiator RTP Content Map has not been set");
1335 }
1336 final SessionDescription offer;
1337 try {
1338 offer = SessionDescription.of(rtpContentMap, true);
1339 } catch (final IllegalArgumentException | NullPointerException e) {
1340 Log.d(
1341 Config.LOGTAG,
1342 id.account.getJid().asBareJid()
1343 + ": unable convert offer from session-initiate to SDP",
1344 e);
1345 webRTCWrapper.close();
1346 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1347 return;
1348 }
1349 sendSessionAccept(rtpContentMap.getMedia(), offer);
1350 }
1351
1352 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1353 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1354 }
1355
1356 private synchronized void sendSessionAccept(
1357 final Set<Media> media,
1358 final SessionDescription offer,
1359 final List<PeerConnection.IceServer> iceServers) {
1360 if (isTerminated()) {
1361 Log.w(
1362 Config.LOGTAG,
1363 id.account.getJid().asBareJid()
1364 + ": ICE servers got discovered when session was already terminated."
1365 + " nothing to do.");
1366 return;
1367 }
1368 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1369 try {
1370 setupWebRTC(media, iceServers, !includeCandidates);
1371 } catch (final WebRTCWrapper.InitializationException e) {
1372 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1373 webRTCWrapper.close();
1374 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1375 return;
1376 }
1377 final org.webrtc.SessionDescription sdp =
1378 new org.webrtc.SessionDescription(
1379 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1380 try {
1381 this.webRTCWrapper.setRemoteDescription(sdp).get();
1382 addIceCandidatesFromBlackLog();
1383 org.webrtc.SessionDescription webRTCSessionDescription =
1384 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1385 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1386 } catch (final Exception e) {
1387 failureToAcceptSession(e);
1388 }
1389 }
1390
1391 private void failureToAcceptSession(final Throwable throwable) {
1392 if (isTerminated()) {
1393 return;
1394 }
1395 final Throwable rootCause = Throwables.getRootCause(throwable);
1396 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1397 webRTCWrapper.close();
1398 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1399 }
1400
1401 private void failureToPerformAction(final Jingle.Action action, final Throwable throwable) {
1402 if (isTerminated()) {
1403 return;
1404 }
1405 final Throwable rootCause = Throwables.getRootCause(throwable);
1406 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1407 webRTCWrapper.close();
1408 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1409 }
1410
1411 private void addIceCandidatesFromBlackLog() {
1412 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1413 while ((foo = this.pendingIceCandidates.poll()) != null) {
1414 processCandidate(foo);
1415 Log.d(
1416 Config.LOGTAG,
1417 id.account.getJid().asBareJid() + ": added candidate from back log");
1418 }
1419 }
1420
1421 private void prepareSessionAccept(
1422 final org.webrtc.SessionDescription webRTCSessionDescription,
1423 final boolean includeCandidates) {
1424 final SessionDescription sessionDescription =
1425 SessionDescription.parse(webRTCSessionDescription.description);
1426 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1427 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1428 if (includeCandidates) {
1429 candidates = parseCandidates(sessionDescription);
1430 } else {
1431 candidates = ImmutableMultimap.of();
1432 }
1433 this.responderRtpContentMap = respondingRtpContentMap;
1434 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1435 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1436 prepareOutgoingContentMap(respondingRtpContentMap);
1437 Futures.addCallback(
1438 outgoingContentMapFuture,
1439 new FutureCallback<>() {
1440 @Override
1441 public void onSuccess(final RtpContentMap outgoingContentMap) {
1442 if (includeCandidates) {
1443 Log.d(
1444 Config.LOGTAG,
1445 "including "
1446 + candidates.size()
1447 + " candidates in session accept");
1448 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1449 } else {
1450 sendSessionAccept(outgoingContentMap);
1451 }
1452 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1453 }
1454
1455 @Override
1456 public void onFailure(@NonNull Throwable throwable) {
1457 failureToAcceptSession(throwable);
1458 }
1459 },
1460 MoreExecutors.directExecutor());
1461 }
1462
1463 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1464 if (isTerminated()) {
1465 Log.w(
1466 Config.LOGTAG,
1467 id.account.getJid().asBareJid()
1468 + ": preparing session accept was too slow. already terminated. nothing"
1469 + " to do.");
1470 return;
1471 }
1472 transitionOrThrow(State.SESSION_ACCEPTED);
1473 final Iq sessionAccept =
1474 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_ACCEPT, id.sessionId);
1475 send(sessionAccept);
1476 }
1477
1478 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1479 final RtpContentMap rtpContentMap) {
1480 if (this.omemoVerification.hasDeviceId()) {
1481 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1482 verifiedPayloadFuture =
1483 id.account
1484 .getAxolotlService()
1485 .encrypt(
1486 rtpContentMap,
1487 id.with,
1488 omemoVerification.getDeviceId());
1489 return Futures.transform(
1490 verifiedPayloadFuture,
1491 verifiedPayload -> {
1492 omemoVerification.setOrEnsureEqual(verifiedPayload);
1493 return verifiedPayload.getPayload();
1494 },
1495 MoreExecutors.directExecutor());
1496 } else {
1497 return Futures.immediateFuture(rtpContentMap);
1498 }
1499 }
1500
1501 synchronized void deliveryMessage(
1502 final Jid from,
1503 final Element message,
1504 final String serverMessageId,
1505 final long timestamp) {
1506 Log.d(
1507 Config.LOGTAG,
1508 id.account.getJid().asBareJid()
1509 + ": delivered message to JingleRtpConnection "
1510 + message);
1511 switch (message.getName()) {
1512 case "propose" ->
1513 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1514 case "proceed" ->
1515 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1516 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1517 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1518 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1519 }
1520 }
1521
1522 void deliverFailedProceed(final String message) {
1523 Log.d(
1524 Config.LOGTAG,
1525 id.account.getJid().asBareJid()
1526 + ": receive message error for proceed message ("
1527 + Strings.nullToEmpty(message)
1528 + ")");
1529 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1530 webRTCWrapper.close();
1531 Log.d(
1532 Config.LOGTAG,
1533 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1534 this.finish();
1535 }
1536 }
1537
1538 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1539 final boolean originatedFromMyself =
1540 from.asBareJid().equals(id.account.getJid().asBareJid());
1541 if (originatedFromMyself) {
1542 if (transition(State.ACCEPTED)) {
1543 acceptedOnOtherDevice(serverMsgId, timestamp);
1544 } else {
1545 Log.d(
1546 Config.LOGTAG,
1547 id.account.getJid().asBareJid()
1548 + ": unable to transition to accept because already in state="
1549 + this.state);
1550 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1551 }
1552 } else {
1553 Log.d(
1554 Config.LOGTAG,
1555 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1556 }
1557 }
1558
1559 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1560 if (serverMsgId != null) {
1561 this.message.setServerMsgId(serverMsgId);
1562 }
1563 this.message.setTime(timestamp);
1564 this.message.setCarbon(true); // indicate that call was accepted on other device
1565 this.writeLogMessageSuccess(0);
1566 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1567 this.finish();
1568 }
1569
1570 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1571 final boolean originatedFromMyself =
1572 from.asBareJid().equals(id.account.getJid().asBareJid());
1573 // reject from another one of my clients
1574 if (originatedFromMyself) {
1575 receiveRejectFromMyself(serverMsgId, timestamp);
1576 } else if (isInitiator()) {
1577 if (from.equals(id.with)) {
1578 receiveRejectFromResponder();
1579 } else {
1580 Log.d(
1581 Config.LOGTAG,
1582 id.account.getJid()
1583 + ": ignoring reject from "
1584 + from
1585 + " for session with "
1586 + id.with);
1587 }
1588 } else {
1589 Log.d(
1590 Config.LOGTAG,
1591 id.account.getJid()
1592 + ": ignoring reject from "
1593 + from
1594 + " for session with "
1595 + id.with);
1596 }
1597 }
1598
1599 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1600 if (transition(State.REJECTED)) {
1601 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1602 this.finish();
1603 if (serverMsgId != null) {
1604 this.message.setServerMsgId(serverMsgId);
1605 }
1606 this.message.setTime(timestamp);
1607 this.message.setCarbon(true); // indicate that call was rejected on other device
1608 writeLogMessageMissed();
1609 } else {
1610 Log.d(
1611 Config.LOGTAG,
1612 "not able to transition into REJECTED because already in " + this.state);
1613 }
1614 }
1615
1616 private void receiveRejectFromResponder() {
1617 if (isInState(State.PROCEED)) {
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": received reject while still in proceed. callee reconsidered");
1622 closeTransitionLogFinish(State.REJECTED_RACED);
1623 return;
1624 }
1625 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1626 Log.d(
1627 Config.LOGTAG,
1628 id.account.getJid()
1629 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee"
1630 + " reconsidered before receiving session-init");
1631 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1632 return;
1633 }
1634 Log.d(
1635 Config.LOGTAG,
1636 id.account.getJid()
1637 + ": ignoring reject from responder because already in state "
1638 + this.state);
1639 }
1640
1641 private void receivePropose(
1642 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1643 final boolean originatedFromMyself =
1644 from.asBareJid().equals(id.account.getJid().asBareJid());
1645 if (originatedFromMyself) {
1646 Log.d(
1647 Config.LOGTAG,
1648 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1649 } else if (transition(
1650 State.PROPOSED,
1651 () -> {
1652 final Collection<RtpDescription> descriptions =
1653 Collections2.transform(
1654 Collections2.filter(
1655 propose.getDescriptions(),
1656 d -> d instanceof RtpDescription),
1657 input -> (RtpDescription) input);
1658 final Collection<Media> media =
1659 Collections2.transform(descriptions, RtpDescription::getMedia);
1660 Preconditions.checkState(
1661 !media.contains(Media.UNKNOWN),
1662 "RTP descriptions contain unknown media");
1663 Log.d(
1664 Config.LOGTAG,
1665 id.account.getJid().asBareJid()
1666 + ": received session proposal from "
1667 + from
1668 + " for "
1669 + media);
1670 this.setProposedMedia(Sets.newHashSet(media));
1671 })) {
1672 if (serverMsgId != null) {
1673 this.message.setServerMsgId(serverMsgId);
1674 }
1675 this.message.setTime(timestamp);
1676 startRinging();
1677 // in environments where we always use discovery timeouts we always want to respond with
1678 // 'ringing'
1679 if (Config.JINGLE_MESSAGE_INIT_STRICT_DEVICE_TIMEOUT
1680 || id.getContact().showInContactList()) {
1681 sendJingleMessage("ringing");
1682 }
1683 } else {
1684 Log.d(
1685 Config.LOGTAG,
1686 id.account.getJid()
1687 + ": ignoring session proposal because already in "
1688 + state);
1689 }
1690 }
1691
1692 private void startRinging() {
1693 this.callIntegration.setRinging();
1694 Log.d(
1695 Config.LOGTAG,
1696 id.account.getJid().asBareJid()
1697 + ": received call from "
1698 + id.with
1699 + ". start ringing");
1700 ringingTimeoutFuture =
1701 jingleConnectionManager.schedule(
1702 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1703 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1704 return;
1705 }
1706 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1707 }
1708
1709 private synchronized void ringingTimeout() {
1710 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1711 switch (this.state) {
1712 case PROPOSED -> {
1713 message.markUnread();
1714 rejectCallFromProposed();
1715 }
1716 case SESSION_INITIALIZED -> {
1717 message.markUnread();
1718 rejectCallFromSessionInitiate();
1719 }
1720 }
1721 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1722 }
1723
1724 private void cancelRingingTimeout() {
1725 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1726 if (future != null && !future.isCancelled()) {
1727 future.cancel(false);
1728 }
1729 }
1730
1731 private void receiveProceed(
1732 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1733 final Set<Media> media =
1734 Preconditions.checkNotNull(
1735 this.proposedMedia, "Proposed media has to be set before handling proceed");
1736 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1737 if (from.equals(id.with)) {
1738 if (isInitiator()) {
1739 if (transition(State.PROCEED)) {
1740 if (serverMsgId != null) {
1741 this.message.setServerMsgId(serverMsgId);
1742 }
1743 this.message.setTime(timestamp);
1744 final Integer remoteDeviceId = proceed.getDeviceId();
1745 if (isOmemoEnabled()) {
1746 this.omemoVerification.setDeviceId(remoteDeviceId);
1747 } else {
1748 if (remoteDeviceId != null) {
1749 Log.d(
1750 Config.LOGTAG,
1751 id.account.getJid().asBareJid()
1752 + ": remote party signaled support for OMEMO"
1753 + " verification but we have OMEMO disabled");
1754 }
1755 this.omemoVerification.setDeviceId(null);
1756 }
1757 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1758 } else {
1759 Log.d(
1760 Config.LOGTAG,
1761 String.format(
1762 "%s: ignoring proceed because already in %s",
1763 id.account.getJid().asBareJid(), this.state));
1764 }
1765 } else {
1766 Log.d(
1767 Config.LOGTAG,
1768 String.format(
1769 "%s: ignoring proceed because we were not initializing",
1770 id.account.getJid().asBareJid()));
1771 }
1772 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1773 if (transition(State.ACCEPTED)) {
1774 Log.d(
1775 Config.LOGTAG,
1776 id.account.getJid().asBareJid()
1777 + ": moved session with "
1778 + id.with
1779 + " into state accepted after received carbon copied proceed");
1780 acceptedOnOtherDevice(serverMsgId, timestamp);
1781 }
1782 } else {
1783 Log.d(
1784 Config.LOGTAG,
1785 String.format(
1786 "%s: ignoring proceed from %s. was expected from %s",
1787 id.account.getJid().asBareJid(), from, id.with));
1788 }
1789 }
1790
1791 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1792 if (from.equals(id.with)) {
1793 final State target =
1794 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1795 if (transition(target)) {
1796 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1797 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1798 Log.d(
1799 Config.LOGTAG,
1800 id.account.getJid().asBareJid()
1801 + ": session with "
1802 + id.with
1803 + " has been retracted (serverMsgId="
1804 + serverMsgId
1805 + ")");
1806 if (serverMsgId != null) {
1807 this.message.setServerMsgId(serverMsgId);
1808 }
1809 this.message.setTime(timestamp);
1810 if (target == State.RETRACTED) {
1811 this.message.markUnread();
1812 }
1813 writeLogMessageMissed();
1814 finish();
1815 } else {
1816 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1817 }
1818 } else {
1819 // TODO parse retract from self
1820 Log.d(
1821 Config.LOGTAG,
1822 id.account.getJid().asBareJid()
1823 + ": received retract from "
1824 + from
1825 + ". expected retract from"
1826 + id.with
1827 + ". ignoring");
1828 }
1829 }
1830
1831 public void sendSessionInitiate() {
1832 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1833 }
1834
1835 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1836 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1837 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1838 }
1839
1840 private synchronized void sendSessionInitiate(
1841 final Set<Media> media,
1842 final State targetState,
1843 final List<PeerConnection.IceServer> iceServers) {
1844 if (isTerminated()) {
1845 Log.w(
1846 Config.LOGTAG,
1847 id.account.getJid().asBareJid()
1848 + ": ICE servers got discovered when session was already terminated."
1849 + " nothing to do.");
1850 return;
1851 }
1852 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1853 try {
1854 setupWebRTC(media, iceServers, !includeCandidates);
1855 } catch (final WebRTCWrapper.InitializationException e) {
1856 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1857 webRTCWrapper.close();
1858 sendRetract(Reason.ofThrowable(e));
1859 return;
1860 }
1861 try {
1862 org.webrtc.SessionDescription webRTCSessionDescription =
1863 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1864 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1865 } catch (final Exception e) {
1866 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1867 // exceptions
1868 failureToInitiateSession(e, targetState);
1869 }
1870 }
1871
1872 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1873 if (isTerminated()) {
1874 return;
1875 }
1876 Log.d(
1877 Config.LOGTAG,
1878 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1879 Throwables.getRootCause(throwable));
1880 webRTCWrapper.close();
1881 final Reason reason = Reason.ofThrowable(throwable);
1882 if (isInState(targetState)) {
1883 sendSessionTerminate(reason, throwable.getMessage());
1884 } else {
1885 sendRetract(reason);
1886 }
1887 }
1888
1889 private void sendRetract(final Reason reason) {
1890 // TODO embed reason into retract
1891 sendJingleMessage("retract", id.with.asBareJid());
1892 transitionOrThrow(reasonToState(reason));
1893 this.finish();
1894 }
1895
1896 private void prepareSessionInitiate(
1897 final org.webrtc.SessionDescription webRTCSessionDescription,
1898 final boolean includeCandidates,
1899 final State targetState) {
1900 final SessionDescription sessionDescription =
1901 SessionDescription.parse(webRTCSessionDescription.description);
1902 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1903 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1904 if (includeCandidates) {
1905 candidates = parseCandidates(sessionDescription);
1906 } else {
1907 candidates = ImmutableMultimap.of();
1908 }
1909 this.initiatorRtpContentMap = rtpContentMap;
1910 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1911 encryptSessionInitiate(rtpContentMap);
1912 Futures.addCallback(
1913 outgoingContentMapFuture,
1914 new FutureCallback<>() {
1915 @Override
1916 public void onSuccess(final RtpContentMap outgoingContentMap) {
1917 if (includeCandidates) {
1918 Log.d(
1919 Config.LOGTAG,
1920 "including "
1921 + candidates.size()
1922 + " candidates in session initiate");
1923 sendSessionInitiate(
1924 outgoingContentMap.withCandidates(candidates), targetState);
1925 } else {
1926 sendSessionInitiate(outgoingContentMap, targetState);
1927 }
1928 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1929 }
1930
1931 @Override
1932 public void onFailure(@NonNull final Throwable throwable) {
1933 failureToInitiateSession(throwable, targetState);
1934 }
1935 },
1936 MoreExecutors.directExecutor());
1937 }
1938
1939 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1940 if (isTerminated()) {
1941 Log.w(
1942 Config.LOGTAG,
1943 id.account.getJid().asBareJid()
1944 + ": preparing session was too slow. already terminated. nothing to"
1945 + " do.");
1946 return;
1947 }
1948 this.transitionOrThrow(targetState);
1949 final Iq sessionInitiate =
1950 rtpContentMap.toJinglePacket(Jingle.Action.SESSION_INITIATE, id.sessionId);
1951 send(sessionInitiate);
1952 }
1953
1954 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1955 final RtpContentMap rtpContentMap) {
1956 if (this.omemoVerification.hasDeviceId()) {
1957 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1958 verifiedPayloadFuture =
1959 id.account
1960 .getAxolotlService()
1961 .encrypt(
1962 rtpContentMap,
1963 id.with,
1964 omemoVerification.getDeviceId());
1965 final ListenableFuture<RtpContentMap> future =
1966 Futures.transform(
1967 verifiedPayloadFuture,
1968 verifiedPayload -> {
1969 omemoVerification.setSessionFingerprint(
1970 verifiedPayload.getFingerprint());
1971 return verifiedPayload.getPayload();
1972 },
1973 MoreExecutors.directExecutor());
1974 if (Config.REQUIRE_RTP_VERIFICATION) {
1975 return future;
1976 }
1977 return Futures.catching(
1978 future,
1979 CryptoFailedException.class,
1980 e -> {
1981 Log.w(
1982 Config.LOGTAG,
1983 id.account.getJid().asBareJid()
1984 + ": unable to use OMEMO DTLS verification on outgoing"
1985 + " session initiate. falling back",
1986 e);
1987 return rtpContentMap;
1988 },
1989 MoreExecutors.directExecutor());
1990 } else {
1991 return Futures.immediateFuture(rtpContentMap);
1992 }
1993 }
1994
1995 protected void sendSessionTerminate(final Reason reason) {
1996 sendSessionTerminate(reason, null);
1997 }
1998
1999 protected void sendSessionTerminate(final Reason reason, final String text) {
2000 sendSessionTerminate(reason, text, this::writeLogMessage);
2001 sendJingleMessageFinish(reason);
2002 }
2003
2004 private void sendTransportInfo(
2005 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2006 final RtpContentMap transportInfo;
2007 try {
2008 final RtpContentMap rtpContentMap =
2009 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2010 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2011 } catch (final Exception e) {
2012 Log.d(
2013 Config.LOGTAG,
2014 id.account.getJid().asBareJid()
2015 + ": unable to prepare transport-info from candidate for content="
2016 + contentName);
2017 return;
2018 }
2019 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2020 send(iq);
2021 }
2022
2023 public RtpEndUserState getEndUserState() {
2024 switch (this.state) {
2025 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2026 if (isInitiator()) {
2027 return RtpEndUserState.RINGING;
2028 } else {
2029 return RtpEndUserState.INCOMING_CALL;
2030 }
2031 }
2032 case PROCEED -> {
2033 if (isInitiator()) {
2034 return RtpEndUserState.RINGING;
2035 } else {
2036 return RtpEndUserState.ACCEPTING_CALL;
2037 }
2038 }
2039 case SESSION_INITIALIZED_PRE_APPROVED -> {
2040 if (isInitiator()) {
2041 return RtpEndUserState.RINGING;
2042 } else {
2043 return RtpEndUserState.CONNECTING;
2044 }
2045 }
2046 case SESSION_ACCEPTED -> {
2047 final ContentAddition ca = getPendingContentAddition();
2048 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2049 return RtpEndUserState.INCOMING_CONTENT_ADD;
2050 }
2051 return getPeerConnectionStateAsEndUserState();
2052 }
2053 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2054 if (isInitiator()) {
2055 return RtpEndUserState.DECLINED_OR_BUSY;
2056 } else {
2057 return RtpEndUserState.ENDED;
2058 }
2059 }
2060 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2061 return RtpEndUserState.ENDED;
2062 }
2063 case RETRACTED_RACED -> {
2064 if (isInitiator()) {
2065 return RtpEndUserState.ENDED;
2066 } else {
2067 return RtpEndUserState.RETRACTED;
2068 }
2069 }
2070 case TERMINATED_CONNECTIVITY_ERROR -> {
2071 return zeroDuration()
2072 ? RtpEndUserState.CONNECTIVITY_ERROR
2073 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2074 }
2075 case TERMINATED_APPLICATION_FAILURE -> {
2076 return RtpEndUserState.APPLICATION_ERROR;
2077 }
2078 case TERMINATED_SECURITY_ERROR -> {
2079 return RtpEndUserState.SECURITY_ERROR;
2080 }
2081 }
2082 throw new IllegalStateException(
2083 String.format("%s has no equivalent EndUserState", this.state));
2084 }
2085
2086 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2087 final PeerConnection.PeerConnectionState state;
2088 try {
2089 state = webRTCWrapper.getState();
2090 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2091 // We usually close the WebRTCWrapper *before* transitioning so we might still
2092 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2093 return RtpEndUserState.ENDING_CALL;
2094 }
2095 return switch (state) {
2096 case CONNECTED -> RtpEndUserState.CONNECTED;
2097 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2098 case CLOSED -> RtpEndUserState.ENDING_CALL;
2099 default ->
2100 zeroDuration()
2101 ? RtpEndUserState.CONNECTIVITY_ERROR
2102 : RtpEndUserState.RECONNECTING;
2103 };
2104 }
2105
2106 private boolean isPeerConnectionConnected() {
2107 try {
2108 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2109 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2110 return false;
2111 }
2112 }
2113
2114 private void updateCallIntegrationState() {
2115 switch (this.state) {
2116 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2117 if (isInitiator()) {
2118 this.callIntegration.setDialing();
2119 } else {
2120 this.callIntegration.setRinging();
2121 }
2122 }
2123 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2124 if (isInitiator()) {
2125 this.callIntegration.setDialing();
2126 } else {
2127 this.callIntegration.setInitialized();
2128 }
2129 }
2130 case SESSION_ACCEPTED -> {
2131 if (isPeerConnectionConnected()) {
2132 this.callIntegration.setActive();
2133 } else {
2134 this.callIntegration.setInitialized();
2135 }
2136 }
2137 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2138 if (isInitiator()) {
2139 this.callIntegration.busy();
2140 } else {
2141 this.callIntegration.rejected();
2142 }
2143 }
2144 case TERMINATED_SUCCESS -> this.callIntegration.success();
2145 case ACCEPTED -> this.callIntegration.accepted();
2146 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT ->
2147 this.callIntegration.retracted();
2148 case TERMINATED_CONNECTIVITY_ERROR,
2149 TERMINATED_APPLICATION_FAILURE,
2150 TERMINATED_SECURITY_ERROR ->
2151 this.callIntegration.error();
2152 default ->
2153 throw new IllegalStateException(String.format("%s is not handled", this.state));
2154 }
2155 }
2156
2157 public ContentAddition getPendingContentAddition() {
2158 final RtpContentMap in = this.incomingContentAdd;
2159 final RtpContentMap out = this.outgoingContentAdd;
2160 if (out != null) {
2161 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2162 } else if (in != null) {
2163 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2164 } else {
2165 return null;
2166 }
2167 }
2168
2169 public Set<Media> getMedia() {
2170 final State current = getState();
2171 if (current == State.NULL) {
2172 if (isInitiator()) {
2173 return Preconditions.checkNotNull(
2174 this.proposedMedia, "RTP connection has not been initialized properly");
2175 }
2176 throw new IllegalStateException("RTP connection has not been initialized yet");
2177 }
2178 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2179 return Preconditions.checkNotNull(
2180 this.proposedMedia, "RTP connection has not been initialized properly");
2181 }
2182 final RtpContentMap localContentMap = getLocalContentMap();
2183 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2184 if (localContentMap != null) {
2185 return localContentMap.getMedia();
2186 } else if (initiatorContentMap != null) {
2187 return initiatorContentMap.getMedia();
2188 } else if (isTerminated()) {
2189 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2190 } else {
2191 return Preconditions.checkNotNull(
2192 this.proposedMedia, "RTP connection has not been initialized properly");
2193 }
2194 }
2195
2196 public boolean isVerified() {
2197 final String fingerprint = this.omemoVerification.getFingerprint();
2198 if (fingerprint == null) {
2199 return false;
2200 }
2201 final FingerprintStatus status =
2202 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2203 return status != null && status.isVerified();
2204 }
2205
2206 public boolean addMedia(final Media media) {
2207 final Set<Media> currentMedia = getMedia();
2208 if (currentMedia.contains(media)) {
2209 throw new IllegalStateException(String.format("%s has already been proposed", media));
2210 }
2211 // TODO add state protection - can only add while ACCEPTED or so
2212 Log.d(Config.LOGTAG, "adding media: " + media);
2213 return webRTCWrapper.addTrack(media);
2214 }
2215
2216 public synchronized void acceptCall() {
2217 switch (this.state) {
2218 case PROPOSED -> {
2219 cancelRingingTimeout();
2220 acceptCallFromProposed();
2221 }
2222 case SESSION_INITIALIZED -> {
2223 cancelRingingTimeout();
2224 acceptCallFromSessionInitialized();
2225 }
2226 case ACCEPTED ->
2227 Log.w(
2228 Config.LOGTAG,
2229 id.account.getJid().asBareJid()
2230 + ": the call has already been accepted with another client."
2231 + " UI was just lagging behind");
2232 case PROCEED, SESSION_ACCEPTED ->
2233 Log.w(
2234 Config.LOGTAG,
2235 id.account.getJid().asBareJid()
2236 + ": the call has already been accepted. user probably double"
2237 + " tapped the UI");
2238 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2239 }
2240 }
2241
2242 public synchronized void rejectCall() {
2243 if (isTerminated()) {
2244 Log.w(
2245 Config.LOGTAG,
2246 id.account.getJid().asBareJid()
2247 + ": received rejectCall() when session has already been terminated."
2248 + " nothing to do");
2249 return;
2250 }
2251 switch (this.state) {
2252 case PROPOSED -> rejectCallFromProposed();
2253 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2254 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2255 }
2256 }
2257
2258 public synchronized void integrationFailure() {
2259 final var state = getState();
2260 if (state == State.PROPOSED) {
2261 Log.e(
2262 Config.LOGTAG,
2263 id.account.getJid().asBareJid()
2264 + ": failed call integration in state proposed");
2265 rejectCallFromProposed();
2266 } else if (state == State.SESSION_INITIALIZED) {
2267 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2268 this.webRTCWrapper.close();
2269 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2270 } else {
2271 throw new IllegalStateException(
2272 String.format("Can not fail integration in state %s", state));
2273 }
2274 }
2275
2276 public synchronized void endCall() {
2277 if (isTerminated()) {
2278 Log.w(
2279 Config.LOGTAG,
2280 id.account.getJid().asBareJid()
2281 + ": received endCall() when session has already been terminated."
2282 + " nothing to do");
2283 return;
2284 }
2285 if (isInState(State.PROPOSED) && isResponder()) {
2286 rejectCallFromProposed();
2287 return;
2288 }
2289 if (isInState(State.PROCEED)) {
2290 if (isInitiator()) {
2291 retractFromProceed();
2292 } else {
2293 rejectCallFromProceed();
2294 }
2295 return;
2296 }
2297 if (isInitiator()
2298 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2299 this.webRTCWrapper.close();
2300 sendSessionTerminate(Reason.CANCEL);
2301 return;
2302 }
2303 if (isInState(State.SESSION_INITIALIZED)) {
2304 rejectCallFromSessionInitiate();
2305 return;
2306 }
2307 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2308 this.webRTCWrapper.close();
2309 sendSessionTerminate(Reason.SUCCESS);
2310 return;
2311 }
2312 if (isInState(
2313 State.TERMINATED_APPLICATION_FAILURE,
2314 State.TERMINATED_CONNECTIVITY_ERROR,
2315 State.TERMINATED_DECLINED_OR_BUSY)) {
2316 Log.d(
2317 Config.LOGTAG,
2318 "ignoring request to end call because already in state " + this.state);
2319 return;
2320 }
2321 throw new IllegalStateException(
2322 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2323 }
2324
2325 private void retractFromProceed() {
2326 Log.d(Config.LOGTAG, "retract from proceed");
2327 this.sendJingleMessage("retract");
2328 closeTransitionLogFinish(State.RETRACTED_RACED);
2329 }
2330
2331 private void closeTransitionLogFinish(final State state) {
2332 this.webRTCWrapper.close();
2333 transitionOrThrow(state);
2334 writeLogMessage(state);
2335 finish();
2336 }
2337
2338 private void setupWebRTC(
2339 final Set<Media> media,
2340 final List<PeerConnection.IceServer> iceServers,
2341 final boolean trickle)
2342 throws WebRTCWrapper.InitializationException {
2343 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2344 this.webRTCWrapper.setup(this.xmppConnectionService);
2345 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2346 // this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2347 this.webRTCWrapper.setMicrophoneEnabledOrThrow(true);
2348 }
2349
2350 private void acceptCallFromProposed() {
2351 transitionOrThrow(State.PROCEED);
2352 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2353 this.callIntegration.startAudioRouting();
2354 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2355 this.sendJingleMessage("proceed");
2356 }
2357
2358 private void rejectCallFromProposed() {
2359 transitionOrThrow(State.REJECTED);
2360 writeLogMessageMissed();
2361 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2362 this.sendJingleMessage("reject");
2363 finish();
2364 }
2365
2366 private void rejectCallFromProceed() {
2367 this.sendJingleMessage("reject");
2368 closeTransitionLogFinish(State.REJECTED_RACED);
2369 }
2370
2371 private void rejectCallFromSessionInitiate() {
2372 webRTCWrapper.close();
2373 sendSessionTerminate(Reason.DECLINE);
2374 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2375 }
2376
2377 private void sendJingleMessage(final String action) {
2378 sendJingleMessage(action, id.with);
2379 }
2380
2381 private void sendJingleMessage(final String action, final Jid to) {
2382 final var messagePacket = new im.conversations.android.xmpp.model.stanza.Message();
2383 messagePacket.setType(
2384 im.conversations.android.xmpp.model.stanza.Message.Type
2385 .CHAT); // we want to carbon copy those
2386 messagePacket.setTo(to);
2387 final Element intent =
2388 messagePacket
2389 .addChild(action, Namespace.JINGLE_MESSAGE)
2390 .setAttribute("id", id.sessionId);
2391 if ("proceed".equals(action)) {
2392 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2393 if (isOmemoEnabled()) {
2394 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2395 final Element device =
2396 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2397 device.setAttribute("id", deviceId);
2398 }
2399 }
2400 messagePacket.addChild("store", "urn:xmpp:hints");
2401 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2402 }
2403
2404 private void sendJingleMessageFinish(final Reason reason) {
2405 final var account = id.getAccount();
2406 final var messagePacket =
2407 xmppConnectionService
2408 .getMessageGenerator()
2409 .sessionFinish(id.with, id.sessionId, reason);
2410 xmppConnectionService.sendMessagePacket(account, messagePacket);
2411 }
2412
2413 private boolean isOmemoEnabled() {
2414 final Conversational conversational = message.getConversation();
2415 if (conversational instanceof Conversation) {
2416 return ((Conversation) conversational).getNextEncryption()
2417 == Message.ENCRYPTION_AXOLOTL;
2418 }
2419 return false;
2420 }
2421
2422 private void acceptCallFromSessionInitialized() {
2423 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2424 this.callIntegration.startAudioRouting();
2425 sendSessionAccept();
2426 }
2427
2428 @Override
2429 protected synchronized boolean transition(final State target, final Runnable runnable) {
2430 if (super.transition(target, runnable)) {
2431 updateEndUserState();
2432 updateOngoingCallNotification();
2433 return true;
2434 } else {
2435 return false;
2436 }
2437 }
2438
2439 @Override
2440 public void onIceCandidate(final IceCandidate iceCandidate) {
2441 final RtpContentMap rtpContentMap =
2442 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2443 final IceUdpTransportInfo.Credentials credentials;
2444 try {
2445 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2446 } catch (final IllegalArgumentException e) {
2447 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2448 return;
2449 }
2450 final String uFrag = credentials.ufrag;
2451 final IceUdpTransportInfo.Candidate candidate =
2452 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2453 if (candidate == null) {
2454 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2455 return;
2456 }
2457 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2458 sendTransportInfo(iceCandidate.sdpMid, candidate);
2459 }
2460
2461 @Override
2462 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2463 Log.d(
2464 Config.LOGTAG,
2465 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2466 this.stateHistory.add(newState);
2467 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2468 this.sessionDuration.start();
2469 updateOngoingCallNotification();
2470 } else if (this.sessionDuration.isRunning()) {
2471 this.sessionDuration.stop();
2472 updateOngoingCallNotification();
2473 }
2474
2475 final boolean neverConnected =
2476 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2477
2478 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2479 if (neverConnected) {
2480 if (isTerminated()) {
2481 Log.d(
2482 Config.LOGTAG,
2483 id.account.getJid().asBareJid()
2484 + ": not sending session-terminate after connectivity error"
2485 + " because session is already in state "
2486 + this.state);
2487 return;
2488 }
2489 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2490 return;
2491 } else {
2492 this.restartIce();
2493 }
2494 }
2495 updateEndUserState();
2496 }
2497
2498 private void restartIce() {
2499 this.stateHistory.clear();
2500 this.webRTCWrapper.restartIceAsync();
2501 }
2502
2503 @Override
2504 public void onRenegotiationNeeded() {
2505 this.webRTCWrapper.execute(this::renegotiate);
2506 }
2507
2508 private synchronized void renegotiate() {
2509 final SessionDescription sessionDescription;
2510 try {
2511 sessionDescription = setLocalSessionDescription();
2512 } catch (final Exception e) {
2513 final Throwable cause = Throwables.getRootCause(e);
2514 webRTCWrapper.close();
2515 if (isTerminated()) {
2516 Log.d(
2517 Config.LOGTAG,
2518 "failed to renegotiate. session was already terminated",
2519 cause);
2520 return;
2521 }
2522 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2523 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2524 return;
2525 }
2526 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2527 final RtpContentMap currentContentMap = getLocalContentMap();
2528 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2529 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2530
2531 Log.d(
2532 Config.LOGTAG,
2533 id.getAccount().getJid().asBareJid()
2534 + ": renegotiate. iceRestart="
2535 + iceRestart
2536 + " content id diff="
2537 + diff);
2538
2539 if (diff.hasModifications() && iceRestart) {
2540 webRTCWrapper.close();
2541 sendSessionTerminate(
2542 Reason.FAILED_APPLICATION,
2543 "WebRTC unexpectedly tried to modify content and transport at once");
2544 return;
2545 }
2546
2547 if (iceRestart) {
2548 initiateIceRestart(rtpContentMap);
2549 return;
2550 } else if (diff.isEmpty()) {
2551 Log.d(
2552 Config.LOGTAG,
2553 "renegotiation. nothing to do. SignalingState="
2554 + this.webRTCWrapper.getSignalingState());
2555 }
2556
2557 if (diff.added.isEmpty()) {
2558 return;
2559 }
2560 modifyLocalContentMap(rtpContentMap);
2561 sendContentAdd(rtpContentMap, diff.added);
2562 }
2563
2564 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2565 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2566 final Iq iq = transportInfo.toJinglePacket(Jingle.Action.TRANSPORT_INFO, id.sessionId);
2567 Log.d(Config.LOGTAG, "initiating ice restart: " + iq);
2568 iq.setTo(id.with);
2569 xmppConnectionService.sendIqPacket(
2570 id.account,
2571 iq,
2572 (response) -> {
2573 if (response.getType() == Iq.Type.RESULT) {
2574 Log.d(Config.LOGTAG, "received success to our ice restart");
2575 setLocalContentMap(rtpContentMap);
2576 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2577 return;
2578 }
2579 if (response.getType() == Iq.Type.ERROR) {
2580 if (isTieBreak(response)) {
2581 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2582 return;
2583 }
2584 handleIqErrorResponse(response);
2585 }
2586 if (response.getType() == Iq.Type.TIMEOUT) {
2587 handleIqTimeoutResponse(response);
2588 }
2589 });
2590 }
2591
2592 private boolean isTieBreak(final Iq response) {
2593 final Element error = response.findChild("error");
2594 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2595 }
2596
2597 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2598 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2599 this.outgoingContentAdd = contentAdd;
2600 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2601 prepareOutgoingContentMap(contentAdd);
2602 Futures.addCallback(
2603 outgoingContentMapFuture,
2604 new FutureCallback<>() {
2605 @Override
2606 public void onSuccess(final RtpContentMap outgoingContentMap) {
2607 sendContentAdd(outgoingContentMap);
2608 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2609 }
2610
2611 @Override
2612 public void onFailure(@NonNull Throwable throwable) {
2613 failureToPerformAction(Jingle.Action.CONTENT_ADD, throwable);
2614 }
2615 },
2616 MoreExecutors.directExecutor());
2617 }
2618
2619 private void sendContentAdd(final RtpContentMap contentAdd) {
2620
2621 final Iq iq = contentAdd.toJinglePacket(Jingle.Action.CONTENT_ADD, id.sessionId);
2622 iq.setTo(id.with);
2623 xmppConnectionService.sendIqPacket(
2624 id.account,
2625 iq,
2626 (response) -> {
2627 if (response.getType() == Iq.Type.RESULT) {
2628 Log.d(
2629 Config.LOGTAG,
2630 id.getAccount().getJid().asBareJid()
2631 + ": received ACK to our content-add");
2632 return;
2633 }
2634 if (response.getType() == Iq.Type.ERROR) {
2635 if (isTieBreak(response)) {
2636 this.outgoingContentAdd = null;
2637 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2638 return;
2639 }
2640 handleIqErrorResponse(response);
2641 }
2642 if (response.getType() == Iq.Type.TIMEOUT) {
2643 handleIqTimeoutResponse(response);
2644 }
2645 });
2646 }
2647
2648 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2649 if (isInitiator()) {
2650 this.initiatorRtpContentMap = rtpContentMap;
2651 } else {
2652 this.responderRtpContentMap = rtpContentMap;
2653 }
2654 }
2655
2656 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2657 if (isInitiator()) {
2658 this.responderRtpContentMap = rtpContentMap;
2659 } else {
2660 this.initiatorRtpContentMap = rtpContentMap;
2661 }
2662 }
2663
2664 // this method is to be used for content map modifications that modify media
2665 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2666 final RtpContentMap activeContents = rtpContentMap.activeContents();
2667 setLocalContentMap(activeContents);
2668 this.callIntegration.setAudioDeviceWhenAvailable(
2669 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2670 updateEndUserState();
2671 }
2672
2673 private SessionDescription setLocalSessionDescription()
2674 throws ExecutionException, InterruptedException {
2675 final org.webrtc.SessionDescription sessionDescription =
2676 this.webRTCWrapper.setLocalDescription(false).get();
2677 return SessionDescription.parse(sessionDescription.description);
2678 }
2679
2680 private void closeWebRTCSessionAfterFailedConnection() {
2681 this.webRTCWrapper.close();
2682 synchronized (this) {
2683 if (isTerminated()) {
2684 Log.d(
2685 Config.LOGTAG,
2686 id.account.getJid().asBareJid()
2687 + ": no need to send session-terminate after failed connection."
2688 + " Other party already did");
2689 return;
2690 }
2691 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2692 }
2693 }
2694
2695 public boolean zeroDuration() {
2696 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2697 }
2698
2699 public long getCallDuration() {
2700 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2701 }
2702
2703 @Override
2704 public CallIntegration getCallIntegration() {
2705 return this.callIntegration;
2706 }
2707
2708 public boolean isMicrophoneEnabled() {
2709 return webRTCWrapper.isMicrophoneEnabled();
2710 }
2711
2712 public boolean setMicrophoneEnabled(final boolean enabled) {
2713 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2714 }
2715
2716 public boolean isVideoEnabled() {
2717 return webRTCWrapper.isVideoEnabled();
2718 }
2719
2720 public void setVideoEnabled(final boolean enabled) {
2721 webRTCWrapper.setVideoEnabled(enabled);
2722 }
2723
2724 public boolean isCameraSwitchable() {
2725 return webRTCWrapper.isCameraSwitchable();
2726 }
2727
2728 public boolean isFrontCamera() {
2729 return webRTCWrapper.isFrontCamera();
2730 }
2731
2732 public ListenableFuture<Boolean> switchCamera() {
2733 return webRTCWrapper.switchCamera();
2734 }
2735
2736 @Override
2737 public synchronized void onCallIntegrationShowIncomingCallUi() {
2738 if (isTerminated()) {
2739 // there might be race conditions with the call integration service invoking this
2740 // callback when the rtp session has already ended.
2741 Log.w(
2742 Config.LOGTAG,
2743 "CallIntegration requested incoming call UI but session was already"
2744 + " terminated");
2745 return;
2746 }
2747 // TODO apparently this can be called too early as well?
2748 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2749 }
2750
2751 @Override
2752 public void onCallIntegrationDisconnect() {
2753 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2754 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2755 rejectCall();
2756 } else {
2757 endCall();
2758 }
2759 }
2760
2761 @Override
2762 public void onCallIntegrationReject() {
2763 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2764 try {
2765 rejectCall();
2766 } catch (final IllegalStateException e) {
2767 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2768 }
2769 }
2770
2771 @Override
2772 public void onCallIntegrationAnswer() {
2773 // we need to start the UI to a) show it and b) be able to ask for permissions
2774 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2775 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2776 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toString());
2777 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString());
2778 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2779 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2780 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2781 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2782 xmppConnectionService.startActivity(intent);
2783 }
2784
2785 @Override
2786 public void onCallIntegrationSilence() {
2787 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2788 }
2789
2790 @Override
2791 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2792 // this is called every time we switch audio devices. Thus it would re-enable a microphone
2793 // that was previous disabled by the user. A proper implementation would probably be to
2794 // track user choice and enable the microphone with a userEnabled() &&
2795 // callIntegration.isMicrophoneEnabled() condition
2796 Log.d(Config.LOGTAG, "ignoring onCallIntegrationMicrophoneEnabled(" + enabled + ")");
2797 // this.webRTCWrapper.setMicrophoneEnabled(enabled);
2798 }
2799
2800 @Override
2801 public void onAudioDeviceChanged(
2802 final CallIntegration.AudioDevice selectedAudioDevice,
2803 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2804 Log.d(
2805 Config.LOGTAG,
2806 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2807 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2808 selectedAudioDevice, availableAudioDevices);
2809 }
2810
2811 private void updateEndUserState() {
2812 final RtpEndUserState endUserState = getEndUserState();
2813 this.updateCallIntegrationState();
2814 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2815 id.account, id.with, id.sessionId, endUserState);
2816 }
2817
2818 private void updateOngoingCallNotification() {
2819 final State state = this.state;
2820 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2821 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2822 && isResponder()) {
2823 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2824 xmppConnectionService.removeOngoingCall();
2825 return;
2826 }
2827 final boolean reconnecting;
2828 if (state == State.SESSION_ACCEPTED) {
2829 reconnecting =
2830 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2831 } else {
2832 reconnecting = false;
2833 }
2834 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2835 } else {
2836 xmppConnectionService.removeOngoingCall();
2837 }
2838 }
2839
2840 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2841 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2842 final Iq request = new Iq(Iq.Type.GET);
2843 request.setTo(id.account.getDomain());
2844 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2845 xmppConnectionService.sendIqPacket(
2846 id.account,
2847 request,
2848 (response) -> {
2849 final var iceServers = IceServers.parse(response);
2850 if (iceServers.isEmpty()) {
2851 Log.w(
2852 Config.LOGTAG,
2853 id.account.getJid().asBareJid()
2854 + ": no ICE server found "
2855 + response);
2856 }
2857 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2858 });
2859 } else {
2860 Log.w(
2861 Config.LOGTAG,
2862 id.account.getJid().asBareJid() + ": has no external service discovery");
2863 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2864 }
2865 }
2866
2867 @Override
2868 protected void terminateTransport() {
2869 this.webRTCWrapper.close();
2870 }
2871
2872 @Override
2873 protected void finish() {
2874 if (isTerminated()) {
2875 this.cancelRingingTimeout();
2876 this.callIntegration.verifyDisconnected();
2877 this.webRTCWrapper.verifyClosed();
2878 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2879 super.finish();
2880 try {
2881 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2882 log.getParentFile().mkdirs();
2883 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2884 } catch (final IOException e) { }
2885 } else {
2886 throw new IllegalStateException(
2887 String.format("Unable to call finish from %s", this.state));
2888 }
2889 }
2890
2891 private void writeLogMessage(final State state) {
2892 final long duration = getCallDuration();
2893 if (state == State.TERMINATED_SUCCESS
2894 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2895 writeLogMessageSuccess(duration);
2896 } else {
2897 writeLogMessageMissed();
2898 }
2899 }
2900
2901 private void writeLogMessageSuccess(final long duration) {
2902 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2903 this.writeMessage();
2904 }
2905
2906 private void writeLogMessageMissed() {
2907 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2908 this.writeMessage();
2909 }
2910
2911 private void writeMessage() {
2912 final Conversational conversational = message.getConversation();
2913 if (conversational instanceof Conversation) {
2914 ((Conversation) conversational).add(this.message);
2915 xmppConnectionService.createMessageAsync(message);
2916 xmppConnectionService.updateConversationUi();
2917 } else {
2918 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2919 }
2920 }
2921
2922 public Optional<VideoTrack> getLocalVideoTrack() {
2923 return webRTCWrapper.getLocalVideoTrack();
2924 }
2925
2926 public Optional<VideoTrack> getRemoteVideoTrack() {
2927 return webRTCWrapper.getRemoteVideoTrack();
2928 }
2929
2930 public EglBase.Context getEglBaseContext() {
2931 return webRTCWrapper.getEglBaseContext();
2932 }
2933
2934 void setProposedMedia(final Set<Media> media) {
2935 this.proposedMedia = media;
2936 this.callIntegration.setVideoState(
2937 Media.audioOnly(media)
2938 ? VideoProfile.STATE_AUDIO_ONLY
2939 : VideoProfile.STATE_BIDIRECTIONAL);
2940 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2941 }
2942
2943 public void fireStateUpdate() {
2944 final RtpEndUserState endUserState = getEndUserState();
2945 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2946 id.account, id.with, id.sessionId, endUserState);
2947 }
2948
2949 public boolean isSwitchToVideoAvailable() {
2950 final boolean prerequisite =
2951 Media.audioOnly(getMedia())
2952 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2953 .contains(getEndUserState());
2954 return prerequisite && remoteHasVideoFeature();
2955 }
2956
2957 private boolean remoteHasVideoFeature() {
2958 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2959 }
2960
2961 private boolean remoteHasSdpOfferAnswer() {
2962 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2963 }
2964
2965 @Override
2966 public Account getAccount() {
2967 return id.account;
2968 }
2969
2970 @Override
2971 public Jid getWith() {
2972 return id.with;
2973 }
2974
2975 @Override
2976 public String getSessionId() {
2977 return id.sessionId;
2978 }
2979
2980 private interface OnIceServersDiscovered {
2981 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2982 }
2983}