1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 final var contact = id.getContact();
122 this.callIntegration.setCallerDisplayName(
123 contact.getDisplayName(), TelecomManager.PRESENTATION_ALLOWED);
124 this.callIntegration.setInitialized();
125 }
126
127 JingleRtpConnection(
128 final JingleConnectionManager jingleConnectionManager,
129 final Id id,
130 final Jid initiator,
131 final CallIntegration callIntegration) {
132 super(jingleConnectionManager, id, initiator);
133 final Conversation conversation =
134 jingleConnectionManager
135 .getXmppConnectionService()
136 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
137 this.message =
138 new Message(
139 conversation,
140 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
141 Message.TYPE_RTP_SESSION,
142 id.sessionId);
143 this.callIntegration = callIntegration;
144 this.callIntegration.setCallback(this);
145 }
146
147 @Override
148 synchronized void deliverPacket(final JinglePacket jinglePacket) {
149 switch (jinglePacket.getAction()) {
150 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
151 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
152 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
153 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
154 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
155 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
156 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
157 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
158 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
159 default -> {
160 respondOk(jinglePacket);
161 Log.d(
162 Config.LOGTAG,
163 String.format(
164 "%s: received unhandled jingle action %s",
165 id.account.getJid().asBareJid(), jinglePacket.getAction()));
166 }
167 }
168 }
169
170 @Override
171 synchronized void notifyRebound() {
172 if (isTerminated()) {
173 return;
174 }
175 webRTCWrapper.close();
176 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
177 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
178 }
179 if (isInState(
180 State.SESSION_INITIALIZED,
181 State.SESSION_INITIALIZED_PRE_APPROVED,
182 State.SESSION_ACCEPTED)) {
183 // we might have already changed resources (full jid) at this point; so this might not
184 // even reach the other party
185 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
186 } else {
187 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
188 finish();
189 }
190 }
191
192 public boolean applyDtmfTone(final String tone) {
193 return webRTCWrapper.applyDtmfTone(tone);
194 }
195
196 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
197 respondOk(jinglePacket);
198 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
199 final State previous = this.state;
200 Log.d(
201 Config.LOGTAG,
202 id.account.getJid().asBareJid()
203 + ": received session terminate reason="
204 + wrapper.reason
205 + "("
206 + Strings.nullToEmpty(wrapper.text)
207 + ") while in state "
208 + previous);
209 if (TERMINATED.contains(previous)) {
210 Log.d(
211 Config.LOGTAG,
212 id.account.getJid().asBareJid()
213 + ": ignoring session terminate because already in "
214 + previous);
215 return;
216 }
217 webRTCWrapper.close();
218 final State target = reasonToState(wrapper.reason);
219 transitionOrThrow(target);
220 writeLogMessage(target);
221 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
222 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
223 }
224 finish();
225 }
226
227 private void receiveTransportInfo(final JinglePacket jinglePacket) {
228 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
229 // INITIALIZED only after transport-info has been received
230 if (isInState(
231 State.NULL,
232 State.PROCEED,
233 State.SESSION_INITIALIZED,
234 State.SESSION_INITIALIZED_PRE_APPROVED,
235 State.SESSION_ACCEPTED)) {
236 final RtpContentMap contentMap;
237 try {
238 contentMap = RtpContentMap.of(jinglePacket);
239 } catch (final IllegalArgumentException | NullPointerException e) {
240 Log.d(
241 Config.LOGTAG,
242 id.account.getJid().asBareJid()
243 + ": improperly formatted contents; ignoring",
244 e);
245 respondOk(jinglePacket);
246 return;
247 }
248 receiveTransportInfo(jinglePacket, contentMap);
249 } else {
250 if (isTerminated()) {
251 respondOk(jinglePacket);
252 Log.d(
253 Config.LOGTAG,
254 id.account.getJid().asBareJid()
255 + ": ignoring out-of-order transport info; we where already terminated");
256 } else {
257 Log.d(
258 Config.LOGTAG,
259 id.account.getJid().asBareJid()
260 + ": received transport info while in state="
261 + this.state);
262 terminateWithOutOfOrder(jinglePacket);
263 }
264 }
265 }
266
267 private void receiveTransportInfo(
268 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
269 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
270 candidates = contentMap.contents.entrySet();
271 final RtpContentMap remote = getRemoteContentMap();
272 final Set<String> remoteContentIds =
273 remote == null ? Collections.emptySet() : remote.contents.keySet();
274 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
275 Log.d(
276 Config.LOGTAG,
277 "received transport-info for unknown contents "
278 + contentMap.contents.keySet()
279 + " (known: "
280 + remoteContentIds
281 + ")");
282 respondOk(jinglePacket);
283 pendingIceCandidates.addAll(candidates);
284 return;
285 }
286 if (this.state != State.SESSION_ACCEPTED) {
287 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
288 respondOk(jinglePacket);
289 pendingIceCandidates.addAll(candidates);
290 return;
291 }
292 // zero candidates + modified credentials are an ICE restart offer
293 if (checkForIceRestart(jinglePacket, contentMap)) {
294 return;
295 }
296 respondOk(jinglePacket);
297 try {
298 processCandidates(candidates);
299 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
300 Log.w(
301 Config.LOGTAG,
302 id.account.getJid().asBareJid()
303 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
304 }
305 }
306
307 private void receiveContentAdd(final JinglePacket jinglePacket) {
308 final RtpContentMap modification;
309 try {
310 modification = RtpContentMap.of(jinglePacket);
311 modification.requireContentDescriptions();
312 } catch (final RuntimeException e) {
313 Log.d(
314 Config.LOGTAG,
315 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
316 Throwables.getRootCause(e));
317 respondOk(jinglePacket);
318 webRTCWrapper.close();
319 sendSessionTerminate(Reason.of(e), e.getMessage());
320 return;
321 }
322 if (isInState(State.SESSION_ACCEPTED)) {
323 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
324 final ListenableFuture<RtpContentMap> future =
325 receiveRtpContentMap(
326 modification,
327 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
328 Futures.addCallback(
329 future,
330 new FutureCallback<>() {
331 @Override
332 public void onSuccess(final RtpContentMap rtpContentMap) {
333 receiveContentAdd(jinglePacket, rtpContentMap);
334 }
335
336 @Override
337 public void onFailure(@NonNull Throwable throwable) {
338 respondOk(jinglePacket);
339 final Throwable rootCause = Throwables.getRootCause(throwable);
340 Log.d(
341 Config.LOGTAG,
342 id.account.getJid().asBareJid()
343 + ": improperly formatted contents in content-add",
344 throwable);
345 webRTCWrapper.close();
346 sendSessionTerminate(
347 Reason.ofThrowable(rootCause), rootCause.getMessage());
348 }
349 },
350 MoreExecutors.directExecutor());
351 } else {
352 terminateWithOutOfOrder(jinglePacket);
353 }
354 }
355
356 private void receiveContentAdd(
357 final JinglePacket jinglePacket, final RtpContentMap modification) {
358 final RtpContentMap remote = getRemoteContentMap();
359 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
360 respondOk(jinglePacket);
361 this.webRTCWrapper.close();
362 sendSessionTerminate(
363 Reason.FAILED_APPLICATION,
364 String.format(
365 "contents with names %s already exists",
366 Joiner.on(", ").join(modification.getNames())));
367 return;
368 }
369 final ContentAddition contentAddition =
370 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
371
372 final RtpContentMap outgoing = this.outgoingContentAdd;
373 final Set<ContentAddition.Summary> outgoingContentAddSummary =
374 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
375
376 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
377 if (isInitiator()) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid()
381 + ": respond with tie break to matching content-add offer");
382 respondWithTieBreak(jinglePacket);
383 } else {
384 Log.d(
385 Config.LOGTAG,
386 id.getAccount().getJid().asBareJid()
387 + ": automatically accept matching content-add offer");
388 acceptContentAdd(contentAddition.summary, modification);
389 }
390 return;
391 }
392
393 // once we can display multiple video tracks we can be more loose with this condition
394 // theoretically it should also be fine to automatically accept audio only contents
395 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
396 Log.d(
397 Config.LOGTAG,
398 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
399 this.incomingContentAdd = modification;
400 respondOk(jinglePacket);
401 updateEndUserState();
402 } else {
403 respondOk(jinglePacket);
404 // TODO do we want to add a reason?
405 rejectContentAdd(modification);
406 }
407 }
408
409 private void receiveContentAccept(final JinglePacket jinglePacket) {
410 final RtpContentMap receivedContentAccept;
411 try {
412 receivedContentAccept = RtpContentMap.of(jinglePacket);
413 receivedContentAccept.requireContentDescriptions();
414 } catch (final RuntimeException e) {
415 Log.d(
416 Config.LOGTAG,
417 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
418 Throwables.getRootCause(e));
419 respondOk(jinglePacket);
420 webRTCWrapper.close();
421 sendSessionTerminate(Reason.of(e), e.getMessage());
422 return;
423 }
424
425 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
426 if (outgoingContentAdd == null) {
427 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
428 terminateWithOutOfOrder(jinglePacket);
429 return;
430 }
431 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
432 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
433 this.outgoingContentAdd = null;
434 respondOk(jinglePacket);
435 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
436 final ListenableFuture<RtpContentMap> future =
437 receiveRtpContentMap(
438 receivedContentAccept,
439 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
440 Futures.addCallback(
441 future,
442 new FutureCallback<>() {
443 @Override
444 public void onSuccess(final RtpContentMap result) {
445 receiveContentAccept(result);
446 }
447
448 @Override
449 public void onFailure(@NonNull final Throwable throwable) {
450 webRTCWrapper.close();
451 sendSessionTerminate(
452 Reason.ofThrowable(throwable), throwable.getMessage());
453 }
454 },
455 MoreExecutors.directExecutor());
456 } else {
457 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
458 terminateWithOutOfOrder(jinglePacket);
459 }
460 }
461
462 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
463 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
464 final RtpContentMap modifiedContentMap =
465 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
466
467 setRemoteContentMap(modifiedContentMap);
468
469 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
470
471 final org.webrtc.SessionDescription sdp =
472 new org.webrtc.SessionDescription(
473 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
474
475 try {
476 this.webRTCWrapper.setRemoteDescription(sdp).get();
477 } catch (final Exception e) {
478 final Throwable cause = Throwables.getRootCause(e);
479 Log.d(
480 Config.LOGTAG,
481 id.getAccount().getJid().asBareJid()
482 + ": unable to set remote description after receiving content-accept",
483 cause);
484 webRTCWrapper.close();
485 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
486 return;
487 }
488 Log.d(
489 Config.LOGTAG,
490 id.getAccount().getJid().asBareJid()
491 + ": remote has accepted content-add "
492 + ContentAddition.summary(receivedContentAccept));
493 processCandidates(receivedContentAccept.contents.entrySet());
494 updateEndUserState();
495 }
496
497 private void receiveContentModify(final JinglePacket jinglePacket) {
498 if (this.state != State.SESSION_ACCEPTED) {
499 terminateWithOutOfOrder(jinglePacket);
500 return;
501 }
502 final Map<String, Content.Senders> modification =
503 Maps.transformEntries(
504 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
505 final boolean isInitiator = isInitiator();
506 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
507 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
508 final Set<String> currentOutgoingMediaIds =
509 currentOutgoing == null
510 ? Collections.emptySet()
511 : currentOutgoing.contents.keySet();
512 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
513 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
514 respondOk(jinglePacket);
515 final RtpContentMap modifiedContentMap;
516 try {
517 modifiedContentMap =
518 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
519 } catch (final IllegalArgumentException e) {
520 webRTCWrapper.close();
521 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
522 return;
523 }
524 this.outgoingContentAdd = modifiedContentMap;
525 Log.d(
526 Config.LOGTAG,
527 id.account.getJid().asBareJid()
528 + ": processed content-modification for pending content-add");
529 } else if (remoteContentMap != null
530 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
531 respondOk(jinglePacket);
532 final RtpContentMap modifiedRemoteContentMap;
533 try {
534 modifiedRemoteContentMap =
535 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
536 } catch (final IllegalArgumentException e) {
537 webRTCWrapper.close();
538 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
539 return;
540 }
541 final SessionDescription offer;
542 try {
543 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
544 } catch (final IllegalArgumentException | NullPointerException e) {
545 Log.d(
546 Config.LOGTAG,
547 id.getAccount().getJid().asBareJid()
548 + ": unable convert offer from content-modify to SDP",
549 e);
550 webRTCWrapper.close();
551 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
552 return;
553 }
554 Log.d(
555 Config.LOGTAG,
556 id.account.getJid().asBareJid() + ": auto accepting content-modification");
557 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
558 } else {
559 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
560 respondWithItemNotFound(jinglePacket);
561 }
562 }
563
564 private void autoAcceptContentModify(
565 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
566 this.setRemoteContentMap(modifiedRemoteContentMap);
567 final org.webrtc.SessionDescription sdp =
568 new org.webrtc.SessionDescription(
569 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
570 try {
571 this.webRTCWrapper.setRemoteDescription(sdp).get();
572 // auto accept is only done when we already have tracks
573 final SessionDescription answer = setLocalSessionDescription();
574 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
575 modifyLocalContentMap(rtpContentMap);
576 // we do not need to send an answer but do we have to resend the candidates currently in
577 // SDP?
578 // resendCandidatesFromSdp(answer);
579 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
580 } catch (final Exception e) {
581 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
582 webRTCWrapper.close();
583 sendSessionTerminate(Reason.FAILED_APPLICATION);
584 }
585 }
586
587 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
588 final SessionDescription answer) {
589 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
590 new ImmutableMultimap.Builder<>();
591 for (final SessionDescription.Media media : answer.media) {
592 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
593 if (Strings.isNullOrEmpty(mid)) {
594 continue;
595 }
596 for (final String sdpCandidate : media.attributes.get("candidate")) {
597 final IceUdpTransportInfo.Candidate candidate =
598 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
599 if (candidate != null) {
600 candidateBuilder.put(mid, candidate);
601 }
602 }
603 }
604 return candidateBuilder.build();
605 }
606
607 private void receiveContentReject(final JinglePacket jinglePacket) {
608 final RtpContentMap receivedContentReject;
609 try {
610 receivedContentReject = RtpContentMap.of(jinglePacket);
611 } catch (final RuntimeException e) {
612 Log.d(
613 Config.LOGTAG,
614 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
615 Throwables.getRootCause(e));
616 respondOk(jinglePacket);
617 this.webRTCWrapper.close();
618 sendSessionTerminate(Reason.of(e), e.getMessage());
619 return;
620 }
621
622 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
623 if (outgoingContentAdd == null) {
624 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
625 terminateWithOutOfOrder(jinglePacket);
626 return;
627 }
628 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
629 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
630 this.outgoingContentAdd = null;
631 respondOk(jinglePacket);
632 Log.d(Config.LOGTAG, jinglePacket.toString());
633 receiveContentReject(ourSummary);
634 } else {
635 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
636 terminateWithOutOfOrder(jinglePacket);
637 }
638 }
639
640 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
641 try {
642 this.webRTCWrapper.removeTrack(Media.VIDEO);
643 final RtpContentMap localContentMap = customRollback();
644 modifyLocalContentMap(localContentMap);
645 } catch (final Exception e) {
646 final Throwable cause = Throwables.getRootCause(e);
647 Log.d(
648 Config.LOGTAG,
649 id.getAccount().getJid().asBareJid()
650 + ": unable to rollback local description after receiving content-reject",
651 cause);
652 webRTCWrapper.close();
653 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
654 return;
655 }
656 Log.d(
657 Config.LOGTAG,
658 id.getAccount().getJid().asBareJid()
659 + ": remote has rejected our content-add "
660 + summary);
661 }
662
663 private void receiveContentRemove(final JinglePacket jinglePacket) {
664 final RtpContentMap receivedContentRemove;
665 try {
666 receivedContentRemove = RtpContentMap.of(jinglePacket);
667 receivedContentRemove.requireContentDescriptions();
668 } catch (final RuntimeException e) {
669 Log.d(
670 Config.LOGTAG,
671 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
672 Throwables.getRootCause(e));
673 respondOk(jinglePacket);
674 this.webRTCWrapper.close();
675 sendSessionTerminate(Reason.of(e), e.getMessage());
676 return;
677 }
678 respondOk(jinglePacket);
679 receiveContentRemove(receivedContentRemove);
680 }
681
682 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
683 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
684 final Set<ContentAddition.Summary> contentAddSummary =
685 incomingContentAdd == null
686 ? Collections.emptySet()
687 : ContentAddition.summary(incomingContentAdd);
688 final Set<ContentAddition.Summary> removeSummary =
689 ContentAddition.summary(receivedContentRemove);
690 if (contentAddSummary.equals(removeSummary)) {
691 this.incomingContentAdd = null;
692 updateEndUserState();
693 } else {
694 webRTCWrapper.close();
695 sendSessionTerminate(
696 Reason.FAILED_APPLICATION,
697 String.format(
698 "%s only supports %s as a means to retract a not yet accepted %s",
699 BuildConfig.APP_NAME,
700 JinglePacket.Action.CONTENT_REMOVE,
701 JinglePacket.Action.CONTENT_ADD));
702 }
703 }
704
705 public synchronized void retractContentAdd() {
706 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
707 if (outgoingContentAdd == null) {
708 throw new IllegalStateException("Not outgoing content add");
709 }
710 try {
711 webRTCWrapper.removeTrack(Media.VIDEO);
712 final RtpContentMap localContentMap = customRollback();
713 modifyLocalContentMap(localContentMap);
714 } catch (final Exception e) {
715 final Throwable cause = Throwables.getRootCause(e);
716 Log.d(
717 Config.LOGTAG,
718 id.getAccount().getJid().asBareJid()
719 + ": unable to rollback local description after trying to retract content-add",
720 cause);
721 webRTCWrapper.close();
722 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
723 return;
724 }
725 this.outgoingContentAdd = null;
726 final JinglePacket retract =
727 outgoingContentAdd
728 .toStub()
729 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
730 this.send(retract);
731 Log.d(
732 Config.LOGTAG,
733 id.getAccount().getJid()
734 + ": retract content-add "
735 + ContentAddition.summary(outgoingContentAdd));
736 }
737
738 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
739 final SessionDescription sdp = setLocalSessionDescription();
740 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
741 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
742 this.webRTCWrapper
743 .setRemoteDescription(
744 new org.webrtc.SessionDescription(
745 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
746 .get();
747 return localRtpContentMap;
748 }
749
750 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
751 final RtpContentMap currentRemote = getRemoteContentMap();
752 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
753 if (diff.isEmpty()) {
754 throw new IllegalStateException(
755 "Unexpected rollback condition. No difference between local and remote");
756 }
757 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
758 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
759 final RtpContentMap nextRemote =
760 currentRemote.addContent(
761 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
762 return SessionDescription.of(nextRemote, isResponder());
763 }
764 throw new IllegalStateException(
765 "Unexpected rollback condition. Senders were not uniformly none");
766 }
767
768 public synchronized void acceptContentAdd(
769 @NonNull final Set<ContentAddition.Summary> contentAddition) {
770 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
771 if (incomingContentAdd == null) {
772 throw new IllegalStateException("No incoming content add");
773 }
774
775 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
776 this.incomingContentAdd = null;
777 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
778 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
779 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
780 Log.d(
781 Config.LOGTAG,
782 "content addition is receive only. we want to upgrade to 'both'");
783 final RtpContentMap modifiedSenders =
784 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
785 final JinglePacket proposedContentModification =
786 modifiedSenders
787 .toStub()
788 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
789 proposedContentModification.setTo(id.with);
790 xmppConnectionService.sendIqPacket(
791 id.account,
792 proposedContentModification,
793 (account, response) -> {
794 if (response.getType() == IqPacket.TYPE.RESULT) {
795 Log.d(
796 Config.LOGTAG,
797 id.account.getJid().asBareJid()
798 + ": remote has accepted our upgrade to senders=both");
799 acceptContentAdd(
800 ContentAddition.summary(modifiedSenders), modifiedSenders);
801 } else {
802 Log.d(
803 Config.LOGTAG,
804 id.account.getJid().asBareJid()
805 + ": remote has rejected our upgrade to senders=both");
806 acceptContentAdd(contentAddition, incomingContentAdd);
807 }
808 });
809 } else {
810 acceptContentAdd(contentAddition, incomingContentAdd);
811 }
812 } else {
813 throw new IllegalStateException(
814 "Accepted content add does not match pending content-add");
815 }
816 }
817
818 private void acceptContentAdd(
819 @NonNull final Set<ContentAddition.Summary> contentAddition,
820 final RtpContentMap incomingContentAdd) {
821 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
822 final RtpContentMap modifiedContentMap =
823 getRemoteContentMap().addContent(incomingContentAdd, setup);
824 this.setRemoteContentMap(modifiedContentMap);
825
826 final SessionDescription offer;
827 try {
828 offer = SessionDescription.of(modifiedContentMap, isResponder());
829 } catch (final IllegalArgumentException | NullPointerException e) {
830 Log.d(
831 Config.LOGTAG,
832 id.getAccount().getJid().asBareJid()
833 + ": unable convert offer from content-add to SDP",
834 e);
835 webRTCWrapper.close();
836 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
837 return;
838 }
839 this.incomingContentAdd = null;
840 acceptContentAdd(contentAddition, offer);
841 }
842
843 private void acceptContentAdd(
844 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
845 final org.webrtc.SessionDescription sdp =
846 new org.webrtc.SessionDescription(
847 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
848 try {
849 this.webRTCWrapper.setRemoteDescription(sdp).get();
850
851 // TODO add tracks for 'media' where contentAddition.senders matches
852
853 // TODO if senders.sending(isInitiator())
854
855 this.webRTCWrapper.addTrack(Media.VIDEO);
856
857 // TODO add additional transceivers for recv only cases
858
859 final SessionDescription answer = setLocalSessionDescription();
860 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
861
862 final RtpContentMap contentAcceptMap =
863 rtpContentMap.toContentModification(
864 Collections2.transform(contentAddition, ca -> ca.name));
865
866 Log.d(
867 Config.LOGTAG,
868 id.getAccount().getJid().asBareJid()
869 + ": sending content-accept "
870 + ContentAddition.summary(contentAcceptMap));
871
872 addIceCandidatesFromBlackLog();
873
874 modifyLocalContentMap(rtpContentMap);
875 final ListenableFuture<RtpContentMap> future =
876 prepareOutgoingContentMap(contentAcceptMap);
877 Futures.addCallback(
878 future,
879 new FutureCallback<>() {
880 @Override
881 public void onSuccess(final RtpContentMap rtpContentMap) {
882 sendContentAccept(rtpContentMap);
883 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
884 }
885
886 @Override
887 public void onFailure(@NonNull final Throwable throwable) {
888 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
889 }
890 },
891 MoreExecutors.directExecutor());
892 } catch (final Exception e) {
893 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
894 webRTCWrapper.close();
895 sendSessionTerminate(Reason.FAILED_APPLICATION);
896 }
897 }
898
899 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
900 final JinglePacket jinglePacket =
901 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
902 send(jinglePacket);
903 }
904
905 public synchronized void rejectContentAdd() {
906 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
907 if (incomingContentAdd == null) {
908 throw new IllegalStateException("No incoming content add");
909 }
910 this.incomingContentAdd = null;
911 updateEndUserState();
912 rejectContentAdd(incomingContentAdd);
913 }
914
915 private void rejectContentAdd(final RtpContentMap contentMap) {
916 final JinglePacket jinglePacket =
917 contentMap
918 .toStub()
919 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
920 Log.d(
921 Config.LOGTAG,
922 id.getAccount().getJid().asBareJid()
923 + ": rejecting content "
924 + ContentAddition.summary(contentMap));
925 send(jinglePacket);
926 }
927
928 private boolean checkForIceRestart(
929 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
930 final RtpContentMap existing = getRemoteContentMap();
931 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
932 final IceUdpTransportInfo.Credentials newCredentials;
933 try {
934 existingCredentials = existing.getCredentials();
935 newCredentials = rtpContentMap.getDistinctCredentials();
936 } catch (final IllegalStateException e) {
937 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
938 return false;
939 }
940 if (existingCredentials.contains(newCredentials)) {
941 return false;
942 }
943 // TODO an alternative approach is to check if we already got an iq result to our
944 // ICE-restart
945 // and if that's the case we are seeing an answer.
946 // This might be more spec compliant but also more error prone potentially
947 final boolean isSignalStateStable =
948 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
949 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
950 final boolean isOffer = rtpContentMap.emptyCandidates();
951 final RtpContentMap restartContentMap;
952 try {
953 if (isOffer) {
954 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
955 restartContentMap =
956 existing.modifiedCredentials(
957 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
958 } else {
959 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
960 Log.d(
961 Config.LOGTAG,
962 "received confirmation of ICE restart"
963 + newCredentials
964 + " peer_setup="
965 + setup);
966 // DTLS setup attribute needs to be rewritten to reflect current peer state
967 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
968 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
969 }
970 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
971 return isOffer;
972 } else {
973 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
974 respondWithTieBreak(jinglePacket);
975 return true;
976 }
977 } catch (final Exception exception) {
978 respondOk(jinglePacket);
979 final Throwable rootCause = Throwables.getRootCause(exception);
980 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
981 // If this happens a termination is already in progress
982 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
983 return true;
984 }
985 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
986 webRTCWrapper.close();
987 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
988 return true;
989 }
990 }
991
992 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
993 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
994 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
995 throw new IllegalStateException("Invalid peer setup");
996 }
997 return peerSetup;
998 }
999
1000 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1001 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1002 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1003 }
1004 this.peerDtlsSetup = setup;
1005 }
1006
1007 private boolean applyIceRestart(
1008 final JinglePacket jinglePacket,
1009 final RtpContentMap restartContentMap,
1010 final boolean isOffer)
1011 throws ExecutionException, InterruptedException {
1012 final SessionDescription sessionDescription =
1013 SessionDescription.of(restartContentMap, isResponder());
1014 final org.webrtc.SessionDescription.Type type =
1015 isOffer
1016 ? org.webrtc.SessionDescription.Type.OFFER
1017 : org.webrtc.SessionDescription.Type.ANSWER;
1018 org.webrtc.SessionDescription sdp =
1019 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1020 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1021 if (isInitiator()) {
1022 // We ignore the offer and respond with tie-break. This will clause the responder
1023 // not to apply the content map
1024 return false;
1025 }
1026 }
1027 webRTCWrapper.setRemoteDescription(sdp).get();
1028 setRemoteContentMap(restartContentMap);
1029 if (isOffer) {
1030 final SessionDescription localSessionDescription = setLocalSessionDescription();
1031 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1032 // We need to respond OK before sending any candidates
1033 respondOk(jinglePacket);
1034 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1035 } else {
1036 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1037 }
1038 return true;
1039 }
1040
1041 private void processCandidates(
1042 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1043 contents) {
1044 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1045 content : contents) {
1046 processCandidate(content);
1047 }
1048 }
1049
1050 private void processCandidate(
1051 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1052 content) {
1053 final RtpContentMap rtpContentMap = getRemoteContentMap();
1054 final List<String> indices = toIdentificationTags(rtpContentMap);
1055 final String sdpMid = content.getKey(); // aka content name
1056 final IceUdpTransportInfo transport = content.getValue().transport;
1057 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1058
1059 // TODO check that credentials remained the same
1060
1061 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1062 final String sdp;
1063 try {
1064 sdp = candidate.toSdpAttribute(credentials.ufrag);
1065 } catch (final IllegalArgumentException e) {
1066 Log.d(
1067 Config.LOGTAG,
1068 id.account.getJid().asBareJid()
1069 + ": ignoring invalid ICE candidate "
1070 + e.getMessage());
1071 continue;
1072 }
1073 final int mLineIndex = indices.indexOf(sdpMid);
1074 if (mLineIndex < 0) {
1075 Log.w(
1076 Config.LOGTAG,
1077 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1078 }
1079 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1080 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1081 this.webRTCWrapper.addIceCandidate(iceCandidate);
1082 }
1083 }
1084
1085 private RtpContentMap getRemoteContentMap() {
1086 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1087 }
1088
1089 private RtpContentMap getLocalContentMap() {
1090 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1091 }
1092
1093 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1094 final Group originalGroup = rtpContentMap.group;
1095 final List<String> identificationTags =
1096 originalGroup == null
1097 ? rtpContentMap.getNames()
1098 : originalGroup.getIdentificationTags();
1099 if (identificationTags.size() == 0) {
1100 Log.w(
1101 Config.LOGTAG,
1102 id.account.getJid().asBareJid()
1103 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1104 }
1105 return identificationTags;
1106 }
1107
1108 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1109 final JinglePacket jinglePacket, final boolean expectVerification) {
1110 try {
1111 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1112 } catch (final Exception e) {
1113 return Futures.immediateFailedFuture(e);
1114 }
1115 }
1116
1117 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1118 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1119 Log.d(
1120 Config.LOGTAG,
1121 "receiveRtpContentMap("
1122 + receivedContentMap.getClass().getSimpleName()
1123 + ",expectVerification="
1124 + expectVerification
1125 + ")");
1126 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1127 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1128 id.account
1129 .getAxolotlService()
1130 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1131 return Futures.transform(
1132 future,
1133 omemoVerifiedPayload -> {
1134 // TODO test if an exception here triggers a correct abort
1135 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1136 Log.d(
1137 Config.LOGTAG,
1138 id.account.getJid().asBareJid()
1139 + ": received verifiable DTLS fingerprint via "
1140 + omemoVerification);
1141 return omemoVerifiedPayload.getPayload();
1142 },
1143 MoreExecutors.directExecutor());
1144 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1145 return Futures.immediateFailedFuture(
1146 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1147 } else {
1148 return Futures.immediateFuture(receivedContentMap);
1149 }
1150 }
1151
1152 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1153 if (isInitiator()) {
1154 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1155 return;
1156 }
1157 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1158 Futures.addCallback(
1159 future,
1160 new FutureCallback<>() {
1161 @Override
1162 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1163 receiveSessionInitiate(jinglePacket, rtpContentMap);
1164 }
1165
1166 @Override
1167 public void onFailure(@NonNull final Throwable throwable) {
1168 respondOk(jinglePacket);
1169 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1170 }
1171 },
1172 MoreExecutors.directExecutor());
1173 }
1174
1175 private void receiveSessionInitiate(
1176 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1177 try {
1178 contentMap.requireContentDescriptions();
1179 contentMap.requireDTLSFingerprint(true);
1180 } catch (final RuntimeException e) {
1181 Log.d(
1182 Config.LOGTAG,
1183 id.account.getJid().asBareJid() + ": improperly formatted contents",
1184 Throwables.getRootCause(e));
1185 respondOk(jinglePacket);
1186 sendSessionTerminate(Reason.of(e), e.getMessage());
1187 return;
1188 }
1189 Log.d(
1190 Config.LOGTAG,
1191 "processing session-init with " + contentMap.contents.size() + " contents");
1192 final State target;
1193 if (this.state == State.PROCEED) {
1194 Preconditions.checkState(
1195 proposedMedia != null && proposedMedia.size() > 0,
1196 "proposed media must be set when processing pre-approved session-initiate");
1197 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1198 sendSessionTerminate(
1199 Reason.SECURITY_ERROR,
1200 String.format(
1201 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1202 this.proposedMedia, contentMap.getMedia()));
1203 return;
1204 }
1205 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1206 } else {
1207 target = State.SESSION_INITIALIZED;
1208 setProposedMedia(contentMap.getMedia());
1209 }
1210 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1211 respondOk(jinglePacket);
1212 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1213 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1214 Log.d(
1215 Config.LOGTAG,
1216 id.account.getJid().asBareJid()
1217 + ": automatically accepting session-initiate");
1218 sendSessionAccept();
1219 } else {
1220 Log.d(
1221 Config.LOGTAG,
1222 id.account.getJid().asBareJid()
1223 + ": received not pre-approved session-initiate. start ringing");
1224 startRinging();
1225 }
1226 } else {
1227 Log.d(
1228 Config.LOGTAG,
1229 String.format(
1230 "%s: received session-initiate while in state %s",
1231 id.account.getJid().asBareJid(), state));
1232 terminateWithOutOfOrder(jinglePacket);
1233 }
1234 }
1235
1236 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1237 if (isResponder()) {
1238 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1239 return;
1240 }
1241 final ListenableFuture<RtpContentMap> future =
1242 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1243 Futures.addCallback(
1244 future,
1245 new FutureCallback<>() {
1246 @Override
1247 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1248 receiveSessionAccept(jinglePacket, rtpContentMap);
1249 }
1250
1251 @Override
1252 public void onFailure(@NonNull final Throwable throwable) {
1253 respondOk(jinglePacket);
1254 Log.d(
1255 Config.LOGTAG,
1256 id.account.getJid().asBareJid()
1257 + ": improperly formatted contents in session-accept",
1258 throwable);
1259 webRTCWrapper.close();
1260 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1261 }
1262 },
1263 MoreExecutors.directExecutor());
1264 }
1265
1266 private void receiveSessionAccept(
1267 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1268 try {
1269 contentMap.requireContentDescriptions();
1270 contentMap.requireDTLSFingerprint();
1271 } catch (final RuntimeException e) {
1272 respondOk(jinglePacket);
1273 Log.d(
1274 Config.LOGTAG,
1275 id.account.getJid().asBareJid()
1276 + ": improperly formatted contents in session-accept",
1277 e);
1278 webRTCWrapper.close();
1279 sendSessionTerminate(Reason.of(e), e.getMessage());
1280 return;
1281 }
1282 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1283 if (!initiatorMedia.equals(contentMap.getMedia())) {
1284 sendSessionTerminate(
1285 Reason.SECURITY_ERROR,
1286 String.format(
1287 "Your session-included included media %s but our session-initiate was %s",
1288 this.proposedMedia, contentMap.getMedia()));
1289 return;
1290 }
1291 Log.d(
1292 Config.LOGTAG,
1293 "processing session-accept with " + contentMap.contents.size() + " contents");
1294 if (transition(State.SESSION_ACCEPTED)) {
1295 respondOk(jinglePacket);
1296 receiveSessionAccept(contentMap);
1297 } else {
1298 Log.d(
1299 Config.LOGTAG,
1300 String.format(
1301 "%s: received session-accept while in state %s",
1302 id.account.getJid().asBareJid(), state));
1303 respondOk(jinglePacket);
1304 }
1305 }
1306
1307 private void receiveSessionAccept(final RtpContentMap contentMap) {
1308 this.responderRtpContentMap = contentMap;
1309 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1310 final SessionDescription sessionDescription;
1311 try {
1312 sessionDescription = SessionDescription.of(contentMap, false);
1313 } catch (final IllegalArgumentException | NullPointerException e) {
1314 Log.d(
1315 Config.LOGTAG,
1316 id.account.getJid().asBareJid()
1317 + ": unable convert offer from session-accept to SDP",
1318 e);
1319 webRTCWrapper.close();
1320 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1321 return;
1322 }
1323 final org.webrtc.SessionDescription answer =
1324 new org.webrtc.SessionDescription(
1325 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1326 try {
1327 this.webRTCWrapper.setRemoteDescription(answer).get();
1328 } catch (final Exception e) {
1329 Log.d(
1330 Config.LOGTAG,
1331 id.account.getJid().asBareJid()
1332 + ": unable to set remote description after receiving session-accept",
1333 Throwables.getRootCause(e));
1334 webRTCWrapper.close();
1335 sendSessionTerminate(
1336 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1337 return;
1338 }
1339 processCandidates(contentMap.contents.entrySet());
1340 }
1341
1342 private void sendSessionAccept() {
1343 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1344 if (rtpContentMap == null) {
1345 throw new IllegalStateException("initiator RTP Content Map has not been set");
1346 }
1347 final SessionDescription offer;
1348 try {
1349 offer = SessionDescription.of(rtpContentMap, true);
1350 } catch (final IllegalArgumentException | NullPointerException e) {
1351 Log.d(
1352 Config.LOGTAG,
1353 id.account.getJid().asBareJid()
1354 + ": unable convert offer from session-initiate to SDP",
1355 e);
1356 webRTCWrapper.close();
1357 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1358 return;
1359 }
1360 sendSessionAccept(rtpContentMap.getMedia(), offer);
1361 }
1362
1363 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1364 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1365 }
1366
1367 private synchronized void sendSessionAccept(
1368 final Set<Media> media,
1369 final SessionDescription offer,
1370 final List<PeerConnection.IceServer> iceServers) {
1371 if (isTerminated()) {
1372 Log.w(
1373 Config.LOGTAG,
1374 id.account.getJid().asBareJid()
1375 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1376 return;
1377 }
1378 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1379 try {
1380 setupWebRTC(media, iceServers, !includeCandidates);
1381 } catch (final WebRTCWrapper.InitializationException e) {
1382 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1383 webRTCWrapper.close();
1384 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1385 return;
1386 }
1387 final org.webrtc.SessionDescription sdp =
1388 new org.webrtc.SessionDescription(
1389 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1390 try {
1391 this.webRTCWrapper.setRemoteDescription(sdp).get();
1392 addIceCandidatesFromBlackLog();
1393 org.webrtc.SessionDescription webRTCSessionDescription =
1394 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1395 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1396 } catch (final Exception e) {
1397 failureToAcceptSession(e);
1398 }
1399 }
1400
1401 private void failureToAcceptSession(final Throwable throwable) {
1402 if (isTerminated()) {
1403 return;
1404 }
1405 final Throwable rootCause = Throwables.getRootCause(throwable);
1406 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1407 webRTCWrapper.close();
1408 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1409 }
1410
1411 private void failureToPerformAction(
1412 final JinglePacket.Action action, final Throwable throwable) {
1413 if (isTerminated()) {
1414 return;
1415 }
1416 final Throwable rootCause = Throwables.getRootCause(throwable);
1417 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1418 webRTCWrapper.close();
1419 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1420 }
1421
1422 private void addIceCandidatesFromBlackLog() {
1423 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1424 while ((foo = this.pendingIceCandidates.poll()) != null) {
1425 processCandidate(foo);
1426 Log.d(
1427 Config.LOGTAG,
1428 id.account.getJid().asBareJid() + ": added candidate from back log");
1429 }
1430 }
1431
1432 private void prepareSessionAccept(
1433 final org.webrtc.SessionDescription webRTCSessionDescription,
1434 final boolean includeCandidates) {
1435 final SessionDescription sessionDescription =
1436 SessionDescription.parse(webRTCSessionDescription.description);
1437 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1438 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1439 if (includeCandidates) {
1440 candidates = parseCandidates(sessionDescription);
1441 } else {
1442 candidates = ImmutableMultimap.of();
1443 }
1444 this.responderRtpContentMap = respondingRtpContentMap;
1445 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1446 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1447 prepareOutgoingContentMap(respondingRtpContentMap);
1448 Futures.addCallback(
1449 outgoingContentMapFuture,
1450 new FutureCallback<>() {
1451 @Override
1452 public void onSuccess(final RtpContentMap outgoingContentMap) {
1453 if (includeCandidates) {
1454 Log.d(
1455 Config.LOGTAG,
1456 "including "
1457 + candidates.size()
1458 + " candidates in session accept");
1459 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1460 } else {
1461 sendSessionAccept(outgoingContentMap);
1462 }
1463 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1464 }
1465
1466 @Override
1467 public void onFailure(@NonNull Throwable throwable) {
1468 failureToAcceptSession(throwable);
1469 }
1470 },
1471 MoreExecutors.directExecutor());
1472 }
1473
1474 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1475 if (isTerminated()) {
1476 Log.w(
1477 Config.LOGTAG,
1478 id.account.getJid().asBareJid()
1479 + ": preparing session accept was too slow. already terminated. nothing to do.");
1480 return;
1481 }
1482 transitionOrThrow(State.SESSION_ACCEPTED);
1483 final JinglePacket sessionAccept =
1484 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1485 send(sessionAccept);
1486 }
1487
1488 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1489 final RtpContentMap rtpContentMap) {
1490 if (this.omemoVerification.hasDeviceId()) {
1491 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1492 verifiedPayloadFuture =
1493 id.account
1494 .getAxolotlService()
1495 .encrypt(
1496 rtpContentMap,
1497 id.with,
1498 omemoVerification.getDeviceId());
1499 return Futures.transform(
1500 verifiedPayloadFuture,
1501 verifiedPayload -> {
1502 omemoVerification.setOrEnsureEqual(verifiedPayload);
1503 return verifiedPayload.getPayload();
1504 },
1505 MoreExecutors.directExecutor());
1506 } else {
1507 return Futures.immediateFuture(rtpContentMap);
1508 }
1509 }
1510
1511 synchronized void deliveryMessage(
1512 final Jid from,
1513 final Element message,
1514 final String serverMessageId,
1515 final long timestamp) {
1516 Log.d(
1517 Config.LOGTAG,
1518 id.account.getJid().asBareJid()
1519 + ": delivered message to JingleRtpConnection "
1520 + message);
1521 switch (message.getName()) {
1522 case "propose" -> receivePropose(
1523 from, Propose.upgrade(message), serverMessageId, timestamp);
1524 case "proceed" -> receiveProceed(
1525 from, Proceed.upgrade(message), serverMessageId, timestamp);
1526 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1527 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1528 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1529 }
1530 }
1531
1532 void deliverFailedProceed(final String message) {
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid()
1536 + ": receive message error for proceed message ("
1537 + Strings.nullToEmpty(message)
1538 + ")");
1539 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1540 webRTCWrapper.close();
1541 Log.d(
1542 Config.LOGTAG,
1543 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1544 this.finish();
1545 }
1546 }
1547
1548 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1549 final boolean originatedFromMyself =
1550 from.asBareJid().equals(id.account.getJid().asBareJid());
1551 if (originatedFromMyself) {
1552 if (transition(State.ACCEPTED)) {
1553 acceptedOnOtherDevice(serverMsgId, timestamp);
1554 } else {
1555 Log.d(
1556 Config.LOGTAG,
1557 id.account.getJid().asBareJid()
1558 + ": unable to transition to accept because already in state="
1559 + this.state);
1560 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1561 }
1562 } else {
1563 Log.d(
1564 Config.LOGTAG,
1565 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1566 }
1567 }
1568
1569 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1570 if (serverMsgId != null) {
1571 this.message.setServerMsgId(serverMsgId);
1572 }
1573 this.message.setTime(timestamp);
1574 this.message.setCarbon(true); // indicate that call was accepted on other device
1575 this.writeLogMessageSuccess(0);
1576 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1577 this.finish();
1578 }
1579
1580 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1581 final boolean originatedFromMyself =
1582 from.asBareJid().equals(id.account.getJid().asBareJid());
1583 // reject from another one of my clients
1584 if (originatedFromMyself) {
1585 receiveRejectFromMyself(serverMsgId, timestamp);
1586 } else if (isInitiator()) {
1587 if (from.equals(id.with)) {
1588 receiveRejectFromResponder();
1589 } else {
1590 Log.d(
1591 Config.LOGTAG,
1592 id.account.getJid()
1593 + ": ignoring reject from "
1594 + from
1595 + " for session with "
1596 + id.with);
1597 }
1598 } else {
1599 Log.d(
1600 Config.LOGTAG,
1601 id.account.getJid()
1602 + ": ignoring reject from "
1603 + from
1604 + " for session with "
1605 + id.with);
1606 }
1607 }
1608
1609 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1610 if (transition(State.REJECTED)) {
1611 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1612 this.finish();
1613 if (serverMsgId != null) {
1614 this.message.setServerMsgId(serverMsgId);
1615 }
1616 this.message.setTime(timestamp);
1617 this.message.setCarbon(true); // indicate that call was rejected on other device
1618 writeLogMessageMissed();
1619 } else {
1620 Log.d(
1621 Config.LOGTAG,
1622 "not able to transition into REJECTED because already in " + this.state);
1623 }
1624 }
1625
1626 private void receiveRejectFromResponder() {
1627 if (isInState(State.PROCEED)) {
1628 Log.d(
1629 Config.LOGTAG,
1630 id.account.getJid()
1631 + ": received reject while still in proceed. callee reconsidered");
1632 closeTransitionLogFinish(State.REJECTED_RACED);
1633 return;
1634 }
1635 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1636 Log.d(
1637 Config.LOGTAG,
1638 id.account.getJid()
1639 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1640 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1641 return;
1642 }
1643 Log.d(
1644 Config.LOGTAG,
1645 id.account.getJid()
1646 + ": ignoring reject from responder because already in state "
1647 + this.state);
1648 }
1649
1650 private void receivePropose(
1651 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1652 final boolean originatedFromMyself =
1653 from.asBareJid().equals(id.account.getJid().asBareJid());
1654 if (originatedFromMyself) {
1655 Log.d(
1656 Config.LOGTAG,
1657 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1658 } else if (transition(
1659 State.PROPOSED,
1660 () -> {
1661 final Collection<RtpDescription> descriptions =
1662 Collections2.transform(
1663 Collections2.filter(
1664 propose.getDescriptions(),
1665 d -> d instanceof RtpDescription),
1666 input -> (RtpDescription) input);
1667 final Collection<Media> media =
1668 Collections2.transform(descriptions, RtpDescription::getMedia);
1669 Preconditions.checkState(
1670 !media.contains(Media.UNKNOWN),
1671 "RTP descriptions contain unknown media");
1672 Log.d(
1673 Config.LOGTAG,
1674 id.account.getJid().asBareJid()
1675 + ": received session proposal from "
1676 + from
1677 + " for "
1678 + media);
1679 this.setProposedMedia(Sets.newHashSet(media));
1680 })) {
1681 if (serverMsgId != null) {
1682 this.message.setServerMsgId(serverMsgId);
1683 }
1684 this.message.setTime(timestamp);
1685 startRinging();
1686 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1687 sendJingleMessage("ringing");
1688 }
1689 } else {
1690 Log.d(
1691 Config.LOGTAG,
1692 id.account.getJid()
1693 + ": ignoring session proposal because already in "
1694 + state);
1695 }
1696 }
1697
1698 private void startRinging() {
1699 this.callIntegration.setRinging();
1700 Log.d(
1701 Config.LOGTAG,
1702 id.account.getJid().asBareJid()
1703 + ": received call from "
1704 + id.with
1705 + ". start ringing");
1706 ringingTimeoutFuture =
1707 jingleConnectionManager.schedule(
1708 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1709 if (CallIntegration.selfManaged(xmppConnectionService)) {
1710 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1711 return;
1712 }
1713 }
1714 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1715 }
1716
1717 private synchronized void ringingTimeout() {
1718 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1719 switch (this.state) {
1720 case PROPOSED -> {
1721 message.markUnread();
1722 rejectCallFromProposed();
1723 }
1724 case SESSION_INITIALIZED -> {
1725 message.markUnread();
1726 rejectCallFromSessionInitiate();
1727 }
1728 }
1729 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1730 }
1731
1732 private void cancelRingingTimeout() {
1733 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1734 if (future != null && !future.isCancelled()) {
1735 future.cancel(false);
1736 }
1737 }
1738
1739 private void receiveProceed(
1740 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1741 final Set<Media> media =
1742 Preconditions.checkNotNull(
1743 this.proposedMedia, "Proposed media has to be set before handling proceed");
1744 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1745 if (from.equals(id.with)) {
1746 if (isInitiator()) {
1747 if (transition(State.PROCEED)) {
1748 if (serverMsgId != null) {
1749 this.message.setServerMsgId(serverMsgId);
1750 }
1751 this.message.setTime(timestamp);
1752 final Integer remoteDeviceId = proceed.getDeviceId();
1753 if (isOmemoEnabled()) {
1754 this.omemoVerification.setDeviceId(remoteDeviceId);
1755 } else {
1756 if (remoteDeviceId != null) {
1757 Log.d(
1758 Config.LOGTAG,
1759 id.account.getJid().asBareJid()
1760 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1761 }
1762 this.omemoVerification.setDeviceId(null);
1763 }
1764 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1765 } else {
1766 Log.d(
1767 Config.LOGTAG,
1768 String.format(
1769 "%s: ignoring proceed because already in %s",
1770 id.account.getJid().asBareJid(), this.state));
1771 }
1772 } else {
1773 Log.d(
1774 Config.LOGTAG,
1775 String.format(
1776 "%s: ignoring proceed because we were not initializing",
1777 id.account.getJid().asBareJid()));
1778 }
1779 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1780 if (transition(State.ACCEPTED)) {
1781 Log.d(
1782 Config.LOGTAG,
1783 id.account.getJid().asBareJid()
1784 + ": moved session with "
1785 + id.with
1786 + " into state accepted after received carbon copied proceed");
1787 acceptedOnOtherDevice(serverMsgId, timestamp);
1788 }
1789 } else {
1790 Log.d(
1791 Config.LOGTAG,
1792 String.format(
1793 "%s: ignoring proceed from %s. was expected from %s",
1794 id.account.getJid().asBareJid(), from, id.with));
1795 }
1796 }
1797
1798 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1799 if (from.equals(id.with)) {
1800 final State target =
1801 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1802 if (transition(target)) {
1803 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1804 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1805 Log.d(
1806 Config.LOGTAG,
1807 id.account.getJid().asBareJid()
1808 + ": session with "
1809 + id.with
1810 + " has been retracted (serverMsgId="
1811 + serverMsgId
1812 + ")");
1813 if (serverMsgId != null) {
1814 this.message.setServerMsgId(serverMsgId);
1815 }
1816 this.message.setTime(timestamp);
1817 if (target == State.RETRACTED) {
1818 this.message.markUnread();
1819 }
1820 writeLogMessageMissed();
1821 finish();
1822 } else {
1823 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1824 }
1825 } else {
1826 // TODO parse retract from self
1827 Log.d(
1828 Config.LOGTAG,
1829 id.account.getJid().asBareJid()
1830 + ": received retract from "
1831 + from
1832 + ". expected retract from"
1833 + id.with
1834 + ". ignoring");
1835 }
1836 }
1837
1838 public void sendSessionInitiate() {
1839 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1840 }
1841
1842 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1843 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1844 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1845 }
1846
1847 private synchronized void sendSessionInitiate(
1848 final Set<Media> media,
1849 final State targetState,
1850 final List<PeerConnection.IceServer> iceServers) {
1851 if (isTerminated()) {
1852 Log.w(
1853 Config.LOGTAG,
1854 id.account.getJid().asBareJid()
1855 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1856 return;
1857 }
1858 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1859 try {
1860 setupWebRTC(media, iceServers, !includeCandidates);
1861 } catch (final WebRTCWrapper.InitializationException e) {
1862 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1863 webRTCWrapper.close();
1864 sendRetract(Reason.ofThrowable(e));
1865 return;
1866 }
1867 try {
1868 org.webrtc.SessionDescription webRTCSessionDescription =
1869 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1870 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1871 } catch (final Exception e) {
1872 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1873 // exceptions
1874 failureToInitiateSession(e, targetState);
1875 }
1876 }
1877
1878 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1879 if (isTerminated()) {
1880 return;
1881 }
1882 Log.d(
1883 Config.LOGTAG,
1884 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1885 Throwables.getRootCause(throwable));
1886 webRTCWrapper.close();
1887 final Reason reason = Reason.ofThrowable(throwable);
1888 if (isInState(targetState)) {
1889 sendSessionTerminate(reason, throwable.getMessage());
1890 } else {
1891 sendRetract(reason);
1892 }
1893 }
1894
1895 private void sendRetract(final Reason reason) {
1896 // TODO embed reason into retract
1897 sendJingleMessage("retract", id.with.asBareJid());
1898 transitionOrThrow(reasonToState(reason));
1899 this.finish();
1900 }
1901
1902 private void prepareSessionInitiate(
1903 final org.webrtc.SessionDescription webRTCSessionDescription,
1904 final boolean includeCandidates,
1905 final State targetState) {
1906 final SessionDescription sessionDescription =
1907 SessionDescription.parse(webRTCSessionDescription.description);
1908 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1909 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1910 if (includeCandidates) {
1911 candidates = parseCandidates(sessionDescription);
1912 } else {
1913 candidates = ImmutableMultimap.of();
1914 }
1915 this.initiatorRtpContentMap = rtpContentMap;
1916 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1917 encryptSessionInitiate(rtpContentMap);
1918 Futures.addCallback(
1919 outgoingContentMapFuture,
1920 new FutureCallback<>() {
1921 @Override
1922 public void onSuccess(final RtpContentMap outgoingContentMap) {
1923 if (includeCandidates) {
1924 Log.d(
1925 Config.LOGTAG,
1926 "including "
1927 + candidates.size()
1928 + " candidates in session initiate");
1929 sendSessionInitiate(
1930 outgoingContentMap.withCandidates(candidates), targetState);
1931 } else {
1932 sendSessionInitiate(outgoingContentMap, targetState);
1933 }
1934 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1935 }
1936
1937 @Override
1938 public void onFailure(@NonNull final Throwable throwable) {
1939 failureToInitiateSession(throwable, targetState);
1940 }
1941 },
1942 MoreExecutors.directExecutor());
1943 }
1944
1945 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1946 if (isTerminated()) {
1947 Log.w(
1948 Config.LOGTAG,
1949 id.account.getJid().asBareJid()
1950 + ": preparing session was too slow. already terminated. nothing to do.");
1951 return;
1952 }
1953 this.transitionOrThrow(targetState);
1954 final JinglePacket sessionInitiate =
1955 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1956 send(sessionInitiate);
1957 }
1958
1959 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1960 final RtpContentMap rtpContentMap) {
1961 if (this.omemoVerification.hasDeviceId()) {
1962 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1963 verifiedPayloadFuture =
1964 id.account
1965 .getAxolotlService()
1966 .encrypt(
1967 rtpContentMap,
1968 id.with,
1969 omemoVerification.getDeviceId());
1970 final ListenableFuture<RtpContentMap> future =
1971 Futures.transform(
1972 verifiedPayloadFuture,
1973 verifiedPayload -> {
1974 omemoVerification.setSessionFingerprint(
1975 verifiedPayload.getFingerprint());
1976 return verifiedPayload.getPayload();
1977 },
1978 MoreExecutors.directExecutor());
1979 if (Config.REQUIRE_RTP_VERIFICATION) {
1980 return future;
1981 }
1982 return Futures.catching(
1983 future,
1984 CryptoFailedException.class,
1985 e -> {
1986 Log.w(
1987 Config.LOGTAG,
1988 id.account.getJid().asBareJid()
1989 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1990 e);
1991 return rtpContentMap;
1992 },
1993 MoreExecutors.directExecutor());
1994 } else {
1995 return Futures.immediateFuture(rtpContentMap);
1996 }
1997 }
1998
1999 protected void sendSessionTerminate(final Reason reason) {
2000 sendSessionTerminate(reason, null);
2001 }
2002
2003 protected void sendSessionTerminate(final Reason reason, final String text) {
2004 sendSessionTerminate(reason, text, this::writeLogMessage);
2005 sendJingleMessageFinish(reason);
2006 }
2007
2008 private void sendTransportInfo(
2009 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2010 final RtpContentMap transportInfo;
2011 try {
2012 final RtpContentMap rtpContentMap =
2013 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2014 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2015 } catch (final Exception e) {
2016 Log.d(
2017 Config.LOGTAG,
2018 id.account.getJid().asBareJid()
2019 + ": unable to prepare transport-info from candidate for content="
2020 + contentName);
2021 return;
2022 }
2023 final JinglePacket jinglePacket =
2024 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2025 send(jinglePacket);
2026 }
2027
2028 public RtpEndUserState getEndUserState() {
2029 switch (this.state) {
2030 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2031 if (isInitiator()) {
2032 return RtpEndUserState.RINGING;
2033 } else {
2034 return RtpEndUserState.INCOMING_CALL;
2035 }
2036 }
2037 case PROCEED -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.RINGING;
2040 } else {
2041 return RtpEndUserState.ACCEPTING_CALL;
2042 }
2043 }
2044 case SESSION_INITIALIZED_PRE_APPROVED -> {
2045 if (isInitiator()) {
2046 return RtpEndUserState.RINGING;
2047 } else {
2048 return RtpEndUserState.CONNECTING;
2049 }
2050 }
2051 case SESSION_ACCEPTED -> {
2052 final ContentAddition ca = getPendingContentAddition();
2053 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2054 return RtpEndUserState.INCOMING_CONTENT_ADD;
2055 }
2056 return getPeerConnectionStateAsEndUserState();
2057 }
2058 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2059 if (isInitiator()) {
2060 return RtpEndUserState.DECLINED_OR_BUSY;
2061 } else {
2062 return RtpEndUserState.ENDED;
2063 }
2064 }
2065 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2066 return RtpEndUserState.ENDED;
2067 }
2068 case RETRACTED_RACED -> {
2069 if (isInitiator()) {
2070 return RtpEndUserState.ENDED;
2071 } else {
2072 return RtpEndUserState.RETRACTED;
2073 }
2074 }
2075 case TERMINATED_CONNECTIVITY_ERROR -> {
2076 return zeroDuration()
2077 ? RtpEndUserState.CONNECTIVITY_ERROR
2078 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2079 }
2080 case TERMINATED_APPLICATION_FAILURE -> {
2081 return RtpEndUserState.APPLICATION_ERROR;
2082 }
2083 case TERMINATED_SECURITY_ERROR -> {
2084 return RtpEndUserState.SECURITY_ERROR;
2085 }
2086 }
2087 throw new IllegalStateException(
2088 String.format("%s has no equivalent EndUserState", this.state));
2089 }
2090
2091 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2092 final PeerConnection.PeerConnectionState state;
2093 try {
2094 state = webRTCWrapper.getState();
2095 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2096 // We usually close the WebRTCWrapper *before* transitioning so we might still
2097 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2098 return RtpEndUserState.ENDING_CALL;
2099 }
2100 return switch (state) {
2101 case CONNECTED -> RtpEndUserState.CONNECTED;
2102 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2103 case CLOSED -> RtpEndUserState.ENDING_CALL;
2104 default -> zeroDuration()
2105 ? RtpEndUserState.CONNECTIVITY_ERROR
2106 : RtpEndUserState.RECONNECTING;
2107 };
2108 }
2109
2110 private boolean isPeerConnectionConnected() {
2111 try {
2112 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2113 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2114 return false;
2115 }
2116 }
2117
2118 private void updateCallIntegrationState() {
2119 switch (this.state) {
2120 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2121 if (isInitiator()) {
2122 this.callIntegration.setDialing();
2123 } else {
2124 this.callIntegration.setRinging();
2125 }
2126 }
2127 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2128 if (isInitiator()) {
2129 this.callIntegration.setDialing();
2130 } else {
2131 this.callIntegration.setInitialized();
2132 }
2133 }
2134 case SESSION_ACCEPTED -> {
2135 if (isPeerConnectionConnected()) {
2136 this.callIntegration.setActive();
2137 } else {
2138 this.callIntegration.setInitialized();
2139 }
2140 }
2141 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2142 if (isInitiator()) {
2143 this.callIntegration.busy();
2144 } else {
2145 this.callIntegration.rejected();
2146 }
2147 }
2148 case TERMINATED_SUCCESS -> this.callIntegration.success();
2149 case ACCEPTED -> this.callIntegration.accepted();
2150 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2151 .retracted();
2152 case TERMINATED_CONNECTIVITY_ERROR,
2153 TERMINATED_APPLICATION_FAILURE,
2154 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2155 default -> throw new IllegalStateException(
2156 String.format("%s is not handled", this.state));
2157 }
2158 }
2159
2160 public ContentAddition getPendingContentAddition() {
2161 final RtpContentMap in = this.incomingContentAdd;
2162 final RtpContentMap out = this.outgoingContentAdd;
2163 if (out != null) {
2164 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2165 } else if (in != null) {
2166 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2167 } else {
2168 return null;
2169 }
2170 }
2171
2172 public Set<Media> getMedia() {
2173 final State current = getState();
2174 if (current == State.NULL) {
2175 if (isInitiator()) {
2176 return Preconditions.checkNotNull(
2177 this.proposedMedia, "RTP connection has not been initialized properly");
2178 }
2179 throw new IllegalStateException("RTP connection has not been initialized yet");
2180 }
2181 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2182 return Preconditions.checkNotNull(
2183 this.proposedMedia, "RTP connection has not been initialized properly");
2184 }
2185 final RtpContentMap localContentMap = getLocalContentMap();
2186 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2187 if (localContentMap != null) {
2188 return localContentMap.getMedia();
2189 } else if (initiatorContentMap != null) {
2190 return initiatorContentMap.getMedia();
2191 } else if (isTerminated()) {
2192 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2193 } else {
2194 return Preconditions.checkNotNull(
2195 this.proposedMedia, "RTP connection has not been initialized properly");
2196 }
2197 }
2198
2199 public boolean isVerified() {
2200 final String fingerprint = this.omemoVerification.getFingerprint();
2201 if (fingerprint == null) {
2202 return false;
2203 }
2204 final FingerprintStatus status =
2205 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2206 return status != null && status.isVerified();
2207 }
2208
2209 public boolean addMedia(final Media media) {
2210 final Set<Media> currentMedia = getMedia();
2211 if (currentMedia.contains(media)) {
2212 throw new IllegalStateException(String.format("%s has already been proposed", media));
2213 }
2214 // TODO add state protection - can only add while ACCEPTED or so
2215 Log.d(Config.LOGTAG, "adding media: " + media);
2216 return webRTCWrapper.addTrack(media);
2217 }
2218
2219 public synchronized void acceptCall() {
2220 switch (this.state) {
2221 case PROPOSED -> {
2222 cancelRingingTimeout();
2223 acceptCallFromProposed();
2224 }
2225 case SESSION_INITIALIZED -> {
2226 cancelRingingTimeout();
2227 acceptCallFromSessionInitialized();
2228 }
2229 case ACCEPTED -> Log.w(
2230 Config.LOGTAG,
2231 id.account.getJid().asBareJid()
2232 + ": the call has already been accepted with another client. UI was just lagging behind");
2233 case PROCEED, SESSION_ACCEPTED -> Log.w(
2234 Config.LOGTAG,
2235 id.account.getJid().asBareJid()
2236 + ": the call has already been accepted. user probably double tapped the UI");
2237 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2238 }
2239 }
2240
2241 public synchronized void rejectCall() {
2242 if (isTerminated()) {
2243 Log.w(
2244 Config.LOGTAG,
2245 id.account.getJid().asBareJid()
2246 + ": received rejectCall() when session has already been terminated. nothing to do");
2247 return;
2248 }
2249 switch (this.state) {
2250 case PROPOSED -> rejectCallFromProposed();
2251 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2252 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2253 }
2254 }
2255
2256 public synchronized void integrationFailure() {
2257 final var state = getState();
2258 if (state == State.PROPOSED) {
2259 Log.e(
2260 Config.LOGTAG,
2261 id.account.getJid().asBareJid()
2262 + ": failed call integration in state proposed");
2263 rejectCallFromProposed();
2264 } else if (state == State.SESSION_INITIALIZED) {
2265 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2266 this.webRTCWrapper.close();
2267 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2268 } else {
2269 throw new IllegalStateException(
2270 String.format("Can not fail integration in state %s", state));
2271 }
2272 }
2273
2274 public synchronized void endCall() {
2275 if (isTerminated()) {
2276 Log.w(
2277 Config.LOGTAG,
2278 id.account.getJid().asBareJid()
2279 + ": received endCall() when session has already been terminated. nothing to do");
2280 return;
2281 }
2282 if (isInState(State.PROPOSED) && isResponder()) {
2283 rejectCallFromProposed();
2284 return;
2285 }
2286 if (isInState(State.PROCEED)) {
2287 if (isInitiator()) {
2288 retractFromProceed();
2289 } else {
2290 rejectCallFromProceed();
2291 }
2292 return;
2293 }
2294 if (isInitiator()
2295 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2296 this.webRTCWrapper.close();
2297 sendSessionTerminate(Reason.CANCEL);
2298 return;
2299 }
2300 if (isInState(State.SESSION_INITIALIZED)) {
2301 rejectCallFromSessionInitiate();
2302 return;
2303 }
2304 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2305 this.webRTCWrapper.close();
2306 sendSessionTerminate(Reason.SUCCESS);
2307 return;
2308 }
2309 if (isInState(
2310 State.TERMINATED_APPLICATION_FAILURE,
2311 State.TERMINATED_CONNECTIVITY_ERROR,
2312 State.TERMINATED_DECLINED_OR_BUSY)) {
2313 Log.d(
2314 Config.LOGTAG,
2315 "ignoring request to end call because already in state " + this.state);
2316 return;
2317 }
2318 throw new IllegalStateException(
2319 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2320 }
2321
2322 private void retractFromProceed() {
2323 Log.d(Config.LOGTAG, "retract from proceed");
2324 this.sendJingleMessage("retract");
2325 closeTransitionLogFinish(State.RETRACTED_RACED);
2326 }
2327
2328 private void closeTransitionLogFinish(final State state) {
2329 this.webRTCWrapper.close();
2330 transitionOrThrow(state);
2331 writeLogMessage(state);
2332 finish();
2333 }
2334
2335 private void setupWebRTC(
2336 final Set<Media> media,
2337 final List<PeerConnection.IceServer> iceServers,
2338 final boolean trickle)
2339 throws WebRTCWrapper.InitializationException {
2340 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2341 this.webRTCWrapper.setup(this.xmppConnectionService);
2342 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2343 this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2344 }
2345
2346 private void acceptCallFromProposed() {
2347 transitionOrThrow(State.PROCEED);
2348 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2349 this.callIntegration.startAudioRouting();
2350 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2351 this.sendJingleMessage("proceed");
2352 }
2353
2354 private void rejectCallFromProposed() {
2355 transitionOrThrow(State.REJECTED);
2356 writeLogMessageMissed();
2357 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2358 this.sendJingleMessage("reject");
2359 finish();
2360 }
2361
2362 private void rejectCallFromProceed() {
2363 this.sendJingleMessage("reject");
2364 closeTransitionLogFinish(State.REJECTED_RACED);
2365 }
2366
2367 private void rejectCallFromSessionInitiate() {
2368 webRTCWrapper.close();
2369 sendSessionTerminate(Reason.DECLINE);
2370 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2371 }
2372
2373 private void sendJingleMessage(final String action) {
2374 sendJingleMessage(action, id.with);
2375 }
2376
2377 private void sendJingleMessage(final String action, final Jid to) {
2378 final MessagePacket messagePacket = new MessagePacket();
2379 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2380 messagePacket.setTo(to);
2381 final Element intent =
2382 messagePacket
2383 .addChild(action, Namespace.JINGLE_MESSAGE)
2384 .setAttribute("id", id.sessionId);
2385 if ("proceed".equals(action)) {
2386 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2387 if (isOmemoEnabled()) {
2388 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2389 final Element device =
2390 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2391 device.setAttribute("id", deviceId);
2392 }
2393 }
2394 messagePacket.addChild("store", "urn:xmpp:hints");
2395 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2396 }
2397
2398 private void sendJingleMessageFinish(final Reason reason) {
2399 final var account = id.getAccount();
2400 final MessagePacket messagePacket =
2401 xmppConnectionService
2402 .getMessageGenerator()
2403 .sessionFinish(id.with, id.sessionId, reason);
2404 xmppConnectionService.sendMessagePacket(account, messagePacket);
2405 }
2406
2407 private boolean isOmemoEnabled() {
2408 final Conversational conversational = message.getConversation();
2409 if (conversational instanceof Conversation) {
2410 return ((Conversation) conversational).getNextEncryption()
2411 == Message.ENCRYPTION_AXOLOTL;
2412 }
2413 return false;
2414 }
2415
2416 private void acceptCallFromSessionInitialized() {
2417 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2418 this.callIntegration.startAudioRouting();
2419 sendSessionAccept();
2420 }
2421
2422 @Override
2423 protected synchronized boolean transition(final State target, final Runnable runnable) {
2424 if (super.transition(target, runnable)) {
2425 updateEndUserState();
2426 updateOngoingCallNotification();
2427 return true;
2428 } else {
2429 return false;
2430 }
2431 }
2432
2433 @Override
2434 public void onIceCandidate(final IceCandidate iceCandidate) {
2435 final RtpContentMap rtpContentMap =
2436 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2437 final IceUdpTransportInfo.Credentials credentials;
2438 try {
2439 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2440 } catch (final IllegalArgumentException e) {
2441 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2442 return;
2443 }
2444 final String uFrag = credentials.ufrag;
2445 final IceUdpTransportInfo.Candidate candidate =
2446 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2447 if (candidate == null) {
2448 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2449 return;
2450 }
2451 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2452 sendTransportInfo(iceCandidate.sdpMid, candidate);
2453 }
2454
2455 @Override
2456 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2457 Log.d(
2458 Config.LOGTAG,
2459 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2460 this.stateHistory.add(newState);
2461 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2462 this.sessionDuration.start();
2463 updateOngoingCallNotification();
2464 } else if (this.sessionDuration.isRunning()) {
2465 this.sessionDuration.stop();
2466 updateOngoingCallNotification();
2467 }
2468
2469 final boolean neverConnected =
2470 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2471
2472 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2473 if (neverConnected) {
2474 if (isTerminated()) {
2475 Log.d(
2476 Config.LOGTAG,
2477 id.account.getJid().asBareJid()
2478 + ": not sending session-terminate after connectivity error because session is already in state "
2479 + this.state);
2480 return;
2481 }
2482 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2483 return;
2484 } else {
2485 this.restartIce();
2486 }
2487 }
2488 updateEndUserState();
2489 }
2490
2491 private void restartIce() {
2492 this.stateHistory.clear();
2493 this.webRTCWrapper.restartIceAsync();
2494 }
2495
2496 @Override
2497 public void onRenegotiationNeeded() {
2498 this.webRTCWrapper.execute(this::renegotiate);
2499 }
2500
2501 private synchronized void renegotiate() {
2502 final SessionDescription sessionDescription;
2503 try {
2504 sessionDescription = setLocalSessionDescription();
2505 } catch (final Exception e) {
2506 final Throwable cause = Throwables.getRootCause(e);
2507 webRTCWrapper.close();
2508 if (isTerminated()) {
2509 Log.d(
2510 Config.LOGTAG,
2511 "failed to renegotiate. session was already terminated",
2512 cause);
2513 return;
2514 }
2515 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2516 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2517 return;
2518 }
2519 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2520 final RtpContentMap currentContentMap = getLocalContentMap();
2521 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2522 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2523
2524 Log.d(
2525 Config.LOGTAG,
2526 id.getAccount().getJid().asBareJid()
2527 + ": renegotiate. iceRestart="
2528 + iceRestart
2529 + " content id diff="
2530 + diff);
2531
2532 if (diff.hasModifications() && iceRestart) {
2533 webRTCWrapper.close();
2534 sendSessionTerminate(
2535 Reason.FAILED_APPLICATION,
2536 "WebRTC unexpectedly tried to modify content and transport at once");
2537 return;
2538 }
2539
2540 if (iceRestart) {
2541 initiateIceRestart(rtpContentMap);
2542 return;
2543 } else if (diff.isEmpty()) {
2544 Log.d(
2545 Config.LOGTAG,
2546 "renegotiation. nothing to do. SignalingState="
2547 + this.webRTCWrapper.getSignalingState());
2548 }
2549
2550 if (diff.added.isEmpty()) {
2551 return;
2552 }
2553 modifyLocalContentMap(rtpContentMap);
2554 sendContentAdd(rtpContentMap, diff.added);
2555 }
2556
2557 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2558 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2559 final JinglePacket jinglePacket =
2560 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2561 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2562 jinglePacket.setTo(id.with);
2563 xmppConnectionService.sendIqPacket(
2564 id.account,
2565 jinglePacket,
2566 (account, response) -> {
2567 if (response.getType() == IqPacket.TYPE.RESULT) {
2568 Log.d(Config.LOGTAG, "received success to our ice restart");
2569 setLocalContentMap(rtpContentMap);
2570 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2571 return;
2572 }
2573 if (response.getType() == IqPacket.TYPE.ERROR) {
2574 if (isTieBreak(response)) {
2575 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2576 return;
2577 }
2578 handleIqErrorResponse(response);
2579 }
2580 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2581 handleIqTimeoutResponse(response);
2582 }
2583 });
2584 }
2585
2586 private boolean isTieBreak(final IqPacket response) {
2587 final Element error = response.findChild("error");
2588 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2589 }
2590
2591 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2592 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2593 this.outgoingContentAdd = contentAdd;
2594 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2595 prepareOutgoingContentMap(contentAdd);
2596 Futures.addCallback(
2597 outgoingContentMapFuture,
2598 new FutureCallback<>() {
2599 @Override
2600 public void onSuccess(final RtpContentMap outgoingContentMap) {
2601 sendContentAdd(outgoingContentMap);
2602 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2603 }
2604
2605 @Override
2606 public void onFailure(@NonNull Throwable throwable) {
2607 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2608 }
2609 },
2610 MoreExecutors.directExecutor());
2611 }
2612
2613 private void sendContentAdd(final RtpContentMap contentAdd) {
2614
2615 final JinglePacket jinglePacket =
2616 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2617 jinglePacket.setTo(id.with);
2618 xmppConnectionService.sendIqPacket(
2619 id.account,
2620 jinglePacket,
2621 (connection, response) -> {
2622 if (response.getType() == IqPacket.TYPE.RESULT) {
2623 Log.d(
2624 Config.LOGTAG,
2625 id.getAccount().getJid().asBareJid()
2626 + ": received ACK to our content-add");
2627 return;
2628 }
2629 if (response.getType() == IqPacket.TYPE.ERROR) {
2630 if (isTieBreak(response)) {
2631 this.outgoingContentAdd = null;
2632 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2633 return;
2634 }
2635 handleIqErrorResponse(response);
2636 }
2637 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2638 handleIqTimeoutResponse(response);
2639 }
2640 });
2641 }
2642
2643 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2644 if (isInitiator()) {
2645 this.initiatorRtpContentMap = rtpContentMap;
2646 } else {
2647 this.responderRtpContentMap = rtpContentMap;
2648 }
2649 }
2650
2651 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2652 if (isInitiator()) {
2653 this.responderRtpContentMap = rtpContentMap;
2654 } else {
2655 this.initiatorRtpContentMap = rtpContentMap;
2656 }
2657 }
2658
2659 // this method is to be used for content map modifications that modify media
2660 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2661 final RtpContentMap activeContents = rtpContentMap.activeContents();
2662 setLocalContentMap(activeContents);
2663 this.callIntegration.setAudioDeviceWhenAvailable(
2664 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2665 updateEndUserState();
2666 }
2667
2668 private SessionDescription setLocalSessionDescription()
2669 throws ExecutionException, InterruptedException {
2670 final org.webrtc.SessionDescription sessionDescription =
2671 this.webRTCWrapper.setLocalDescription(false).get();
2672 return SessionDescription.parse(sessionDescription.description);
2673 }
2674
2675 private void closeWebRTCSessionAfterFailedConnection() {
2676 this.webRTCWrapper.close();
2677 synchronized (this) {
2678 if (isTerminated()) {
2679 Log.d(
2680 Config.LOGTAG,
2681 id.account.getJid().asBareJid()
2682 + ": no need to send session-terminate after failed connection. Other party already did");
2683 return;
2684 }
2685 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2686 }
2687 }
2688
2689 public boolean zeroDuration() {
2690 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2691 }
2692
2693 public long getCallDuration() {
2694 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2695 }
2696
2697 @Override
2698 public CallIntegration getCallIntegration() {
2699 return this.callIntegration;
2700 }
2701
2702 public boolean isMicrophoneEnabled() {
2703 return webRTCWrapper.isMicrophoneEnabled();
2704 }
2705
2706 public boolean setMicrophoneEnabled(final boolean enabled) {
2707 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2708 }
2709
2710 public boolean isVideoEnabled() {
2711 return webRTCWrapper.isVideoEnabled();
2712 }
2713
2714 public void setVideoEnabled(final boolean enabled) {
2715 webRTCWrapper.setVideoEnabled(enabled);
2716 }
2717
2718 public boolean isCameraSwitchable() {
2719 return webRTCWrapper.isCameraSwitchable();
2720 }
2721
2722 public boolean isFrontCamera() {
2723 return webRTCWrapper.isFrontCamera();
2724 }
2725
2726 public ListenableFuture<Boolean> switchCamera() {
2727 return webRTCWrapper.switchCamera();
2728 }
2729
2730 @Override
2731 public synchronized void onCallIntegrationShowIncomingCallUi() {
2732 if (isTerminated()) {
2733 // there might be race conditions with the call integration service invoking this
2734 // callback when the rtp session has already ended.
2735 Log.w(
2736 Config.LOGTAG,
2737 "CallIntegration requested incoming call UI but session was already terminated");
2738 return;
2739 }
2740 // TODO apparently this can be called too early as well?
2741 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2742 }
2743
2744 @Override
2745 public void onCallIntegrationDisconnect() {
2746 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2747 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2748 rejectCall();
2749 } else {
2750 endCall();
2751 }
2752 }
2753
2754 @Override
2755 public void onCallIntegrationReject() {
2756 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2757 try {
2758 rejectCall();
2759 } catch (final IllegalStateException e) {
2760 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2761 }
2762 }
2763
2764 @Override
2765 public void onCallIntegrationAnswer() {
2766 // we need to start the UI to a) show it and b) be able to ask for permissions
2767 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2768 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2769 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2770 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2771 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2772 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2773 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2774 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2775 xmppConnectionService.startActivity(intent);
2776 }
2777
2778 @Override
2779 public void onCallIntegrationSilence() {
2780 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2781 }
2782
2783 @Override
2784 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2785 this.webRTCWrapper.setMicrophoneEnabled(enabled);
2786 }
2787
2788 @Override
2789 public void onAudioDeviceChanged(
2790 final CallIntegration.AudioDevice selectedAudioDevice,
2791 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2792 Log.d(
2793 Config.LOGTAG,
2794 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2795 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2796 selectedAudioDevice, availableAudioDevices);
2797 }
2798
2799 private void updateEndUserState() {
2800 final RtpEndUserState endUserState = getEndUserState();
2801 this.updateCallIntegrationState();
2802 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2803 id.account, id.with, id.sessionId, endUserState);
2804 }
2805
2806 private void updateOngoingCallNotification() {
2807 final State state = this.state;
2808 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2809 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2810 && isResponder()) {
2811 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2812 xmppConnectionService.removeOngoingCall();
2813 return;
2814 }
2815 final boolean reconnecting;
2816 if (state == State.SESSION_ACCEPTED) {
2817 reconnecting =
2818 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2819 } else {
2820 reconnecting = false;
2821 }
2822 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2823 } else {
2824 xmppConnectionService.removeOngoingCall();
2825 }
2826 }
2827
2828 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2829 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2830 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2831 request.setTo(id.account.getDomain());
2832 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2833 xmppConnectionService.sendIqPacket(
2834 id.account,
2835 request,
2836 (account, response) -> {
2837 final var iceServers = IceServers.parse(response);
2838 if (iceServers.isEmpty()) {
2839 Log.w(
2840 Config.LOGTAG,
2841 id.account.getJid().asBareJid()
2842 + ": no ICE server found "
2843 + response);
2844 }
2845 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2846 });
2847 } else {
2848 Log.w(
2849 Config.LOGTAG,
2850 id.account.getJid().asBareJid() + ": has no external service discovery");
2851 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2852 }
2853 }
2854
2855 @Override
2856 protected void terminateTransport() {
2857 this.webRTCWrapper.close();
2858 }
2859
2860 @Override
2861 protected void finish() {
2862 if (isTerminated()) {
2863 this.cancelRingingTimeout();
2864 this.callIntegration.verifyDisconnected();
2865 this.webRTCWrapper.verifyClosed();
2866 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2867 super.finish();
2868 try {
2869 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2870 log.getParentFile().mkdirs();
2871 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2872 } catch (final IOException e) { }
2873 } else {
2874 throw new IllegalStateException(
2875 String.format("Unable to call finish from %s", this.state));
2876 }
2877 }
2878
2879 private void writeLogMessage(final State state) {
2880 final long duration = getCallDuration();
2881 if (state == State.TERMINATED_SUCCESS
2882 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2883 writeLogMessageSuccess(duration);
2884 } else {
2885 writeLogMessageMissed();
2886 }
2887 }
2888
2889 private void writeLogMessageSuccess(final long duration) {
2890 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2891 this.writeMessage();
2892 }
2893
2894 private void writeLogMessageMissed() {
2895 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2896 this.writeMessage();
2897 }
2898
2899 private void writeMessage() {
2900 final Conversational conversational = message.getConversation();
2901 if (conversational instanceof Conversation) {
2902 ((Conversation) conversational).add(this.message);
2903 xmppConnectionService.createMessageAsync(message);
2904 xmppConnectionService.updateConversationUi();
2905 } else {
2906 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2907 }
2908 }
2909
2910 public Optional<VideoTrack> getLocalVideoTrack() {
2911 return webRTCWrapper.getLocalVideoTrack();
2912 }
2913
2914 public Optional<VideoTrack> getRemoteVideoTrack() {
2915 return webRTCWrapper.getRemoteVideoTrack();
2916 }
2917
2918 public EglBase.Context getEglBaseContext() {
2919 return webRTCWrapper.getEglBaseContext();
2920 }
2921
2922 void setProposedMedia(final Set<Media> media) {
2923 this.proposedMedia = media;
2924 this.callIntegration.setVideoState(
2925 Media.audioOnly(media)
2926 ? VideoProfile.STATE_AUDIO_ONLY
2927 : VideoProfile.STATE_BIDIRECTIONAL);
2928 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2929 }
2930
2931 public void fireStateUpdate() {
2932 final RtpEndUserState endUserState = getEndUserState();
2933 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2934 id.account, id.with, id.sessionId, endUserState);
2935 }
2936
2937 public boolean isSwitchToVideoAvailable() {
2938 final boolean prerequisite =
2939 Media.audioOnly(getMedia())
2940 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2941 .contains(getEndUserState());
2942 return prerequisite && remoteHasVideoFeature();
2943 }
2944
2945 private boolean remoteHasVideoFeature() {
2946 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2947 }
2948
2949 private boolean remoteHasSdpOfferAnswer() {
2950 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2951 }
2952
2953 @Override
2954 public Account getAccount() {
2955 return id.account;
2956 }
2957
2958 @Override
2959 public Jid getWith() {
2960 return id.with;
2961 }
2962
2963 @Override
2964 public String getSessionId() {
2965 return id.sessionId;
2966 }
2967
2968 private interface OnIceServersDiscovered {
2969 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2970 }
2971}