1package eu.siacs.conversations.xmpp.jingle;
2
3import android.telecom.Call;
4import android.telecom.TelecomManager;
5import android.util.Log;
6
7import androidx.annotation.NonNull;
8import androidx.annotation.Nullable;
9
10import com.google.common.base.Joiner;
11import com.google.common.base.Optional;
12import com.google.common.base.Preconditions;
13import com.google.common.base.Stopwatch;
14import com.google.common.base.Strings;
15import com.google.common.base.Throwables;
16import com.google.common.collect.Collections2;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Sets;
22import com.google.common.util.concurrent.FutureCallback;
23import com.google.common.util.concurrent.Futures;
24import com.google.common.util.concurrent.ListenableFuture;
25import com.google.common.util.concurrent.MoreExecutors;
26
27import eu.siacs.conversations.BuildConfig;
28import eu.siacs.conversations.Config;
29import eu.siacs.conversations.crypto.axolotl.AxolotlService;
30import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
31import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
32import eu.siacs.conversations.entities.Conversation;
33import eu.siacs.conversations.entities.Conversational;
34import eu.siacs.conversations.entities.Message;
35import eu.siacs.conversations.entities.RtpSessionStatus;
36import eu.siacs.conversations.services.AppRTCAudioManager;
37import eu.siacs.conversations.services.CallIntegration;
38import eu.siacs.conversations.xml.Element;
39import eu.siacs.conversations.xml.Namespace;
40import eu.siacs.conversations.xmpp.Jid;
41import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
42import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
43import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
44import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
48import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
49import eu.siacs.conversations.xmpp.stanzas.IqPacket;
50import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
51
52import org.webrtc.EglBase;
53import org.webrtc.IceCandidate;
54import org.webrtc.PeerConnection;
55import org.webrtc.VideoTrack;
56
57import java.util.Arrays;
58import java.util.Collection;
59import java.util.Collections;
60import java.util.LinkedList;
61import java.util.List;
62import java.util.Map;
63import java.util.Queue;
64import java.util.Set;
65import java.util.concurrent.ExecutionException;
66import java.util.concurrent.ScheduledFuture;
67import java.util.concurrent.TimeUnit;
68
69public class JingleRtpConnection extends AbstractJingleConnection
70 implements WebRTCWrapper.EventCallback, CallIntegration.Callback {
71
72 public static final List<State> STATES_SHOWING_ONGOING_CALL =
73 Arrays.asList(
74 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
75 private static final long BUSY_TIME_OUT = 30;
76
77 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
78 private final Queue<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>>
79 pendingIceCandidates = new LinkedList<>();
80 private final OmemoVerification omemoVerification = new OmemoVerification();
81 private final CallIntegration callIntegration;
82 private final Message message;
83
84 private Set<Media> proposedMedia;
85 private RtpContentMap initiatorRtpContentMap;
86 private RtpContentMap responderRtpContentMap;
87 private RtpContentMap incomingContentAdd;
88 private RtpContentMap outgoingContentAdd;
89 private IceUdpTransportInfo.Setup peerDtlsSetup;
90 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
91 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
92 private ScheduledFuture<?> ringingTimeoutFuture;
93
94 JingleRtpConnection(final JingleConnectionManager jingleConnectionManager, final Id id, final Jid initiator) {
95 this(jingleConnectionManager, id, initiator, new CallIntegration(jingleConnectionManager.getXmppConnectionService().getApplicationContext()));
96 this.callIntegration.setAddress(CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
97 this.callIntegration.setInitialized();
98 }
99
100 JingleRtpConnection(final JingleConnectionManager jingleConnectionManager, final Id id, final Jid initiator, final CallIntegration callIntegration) {
101 super(jingleConnectionManager, id, initiator);
102 final Conversation conversation =
103 jingleConnectionManager
104 .getXmppConnectionService()
105 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
106 this.message =
107 new Message(
108 conversation,
109 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
110 Message.TYPE_RTP_SESSION,
111 id.sessionId);
112 this.callIntegration = callIntegration;
113 this.callIntegration.setCallback(this);
114 }
115
116 @Override
117 synchronized void deliverPacket(final JinglePacket jinglePacket) {
118 switch (jinglePacket.getAction()) {
119 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
120 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
121 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
122 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
123 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
124 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
125 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
126 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
127 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
128 default -> {
129 respondOk(jinglePacket);
130 Log.d(
131 Config.LOGTAG,
132 String.format(
133 "%s: received unhandled jingle action %s",
134 id.account.getJid().asBareJid(), jinglePacket.getAction()));
135 }
136 }
137 }
138
139 @Override
140 synchronized void notifyRebound() {
141 if (isTerminated()) {
142 return;
143 }
144 webRTCWrapper.close();
145 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
146 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
147 }
148 if (isInState(
149 State.SESSION_INITIALIZED,
150 State.SESSION_INITIALIZED_PRE_APPROVED,
151 State.SESSION_ACCEPTED)) {
152 // we might have already changed resources (full jid) at this point; so this might not
153 // even reach the other party
154 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
155 } else {
156 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
157 finish();
158 }
159 }
160
161 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
162 respondOk(jinglePacket);
163 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
164 final State previous = this.state;
165 Log.d(
166 Config.LOGTAG,
167 id.account.getJid().asBareJid()
168 + ": received session terminate reason="
169 + wrapper.reason
170 + "("
171 + Strings.nullToEmpty(wrapper.text)
172 + ") while in state "
173 + previous);
174 if (TERMINATED.contains(previous)) {
175 Log.d(
176 Config.LOGTAG,
177 id.account.getJid().asBareJid()
178 + ": ignoring session terminate because already in "
179 + previous);
180 return;
181 }
182 webRTCWrapper.close();
183 final State target = reasonToState(wrapper.reason);
184 transitionOrThrow(target);
185 writeLogMessage(target);
186 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
187 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
188 }
189 finish();
190 }
191
192 private void receiveTransportInfo(final JinglePacket jinglePacket) {
193 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
194 // INITIALIZED only after transport-info has been received
195 if (isInState(
196 State.NULL,
197 State.PROCEED,
198 State.SESSION_INITIALIZED,
199 State.SESSION_INITIALIZED_PRE_APPROVED,
200 State.SESSION_ACCEPTED)) {
201 final RtpContentMap contentMap;
202 try {
203 contentMap = RtpContentMap.of(jinglePacket);
204 } catch (final IllegalArgumentException | NullPointerException e) {
205 Log.d(
206 Config.LOGTAG,
207 id.account.getJid().asBareJid()
208 + ": improperly formatted contents; ignoring",
209 e);
210 respondOk(jinglePacket);
211 return;
212 }
213 receiveTransportInfo(jinglePacket, contentMap);
214 } else {
215 if (isTerminated()) {
216 respondOk(jinglePacket);
217 Log.d(
218 Config.LOGTAG,
219 id.account.getJid().asBareJid()
220 + ": ignoring out-of-order transport info; we where already terminated");
221 } else {
222 Log.d(
223 Config.LOGTAG,
224 id.account.getJid().asBareJid()
225 + ": received transport info while in state="
226 + this.state);
227 terminateWithOutOfOrder(jinglePacket);
228 }
229 }
230 }
231
232 private void receiveTransportInfo(
233 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
234 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> candidates =
235 contentMap.contents.entrySet();
236 final RtpContentMap remote = getRemoteContentMap();
237 final Set<String> remoteContentIds =
238 remote == null ? Collections.emptySet() : remote.contents.keySet();
239 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
240 Log.d(
241 Config.LOGTAG,
242 "received transport-info for unknown contents "
243 + contentMap.contents.keySet()
244 + " (known: "
245 + remoteContentIds
246 + ")");
247 respondOk(jinglePacket);
248 pendingIceCandidates.addAll(candidates);
249 return;
250 }
251 if (this.state != State.SESSION_ACCEPTED) {
252 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
253 respondOk(jinglePacket);
254 pendingIceCandidates.addAll(candidates);
255 return;
256 }
257 // zero candidates + modified credentials are an ICE restart offer
258 if (checkForIceRestart(jinglePacket, contentMap)) {
259 return;
260 }
261 respondOk(jinglePacket);
262 try {
263 processCandidates(candidates);
264 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
265 Log.w(
266 Config.LOGTAG,
267 id.account.getJid().asBareJid()
268 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
269 }
270 }
271
272 private void receiveContentAdd(final JinglePacket jinglePacket) {
273 final RtpContentMap modification;
274 try {
275 modification = RtpContentMap.of(jinglePacket);
276 modification.requireContentDescriptions();
277 } catch (final RuntimeException e) {
278 Log.d(
279 Config.LOGTAG,
280 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
281 Throwables.getRootCause(e));
282 respondOk(jinglePacket);
283 webRTCWrapper.close();
284 sendSessionTerminate(Reason.of(e), e.getMessage());
285 return;
286 }
287 if (isInState(State.SESSION_ACCEPTED)) {
288 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
289 final ListenableFuture<RtpContentMap> future =
290 receiveRtpContentMap(
291 modification,
292 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
293 Futures.addCallback(
294 future,
295 new FutureCallback<>() {
296 @Override
297 public void onSuccess(final RtpContentMap rtpContentMap) {
298 receiveContentAdd(jinglePacket, rtpContentMap);
299 }
300
301 @Override
302 public void onFailure(@NonNull Throwable throwable) {
303 respondOk(jinglePacket);
304 final Throwable rootCause = Throwables.getRootCause(throwable);
305 Log.d(
306 Config.LOGTAG,
307 id.account.getJid().asBareJid()
308 + ": improperly formatted contents in content-add",
309 throwable);
310 webRTCWrapper.close();
311 sendSessionTerminate(
312 Reason.ofThrowable(rootCause), rootCause.getMessage());
313 }
314 },
315 MoreExecutors.directExecutor());
316 } else {
317 terminateWithOutOfOrder(jinglePacket);
318 }
319 }
320
321 private void receiveContentAdd(
322 final JinglePacket jinglePacket, final RtpContentMap modification) {
323 final RtpContentMap remote = getRemoteContentMap();
324 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
325 respondOk(jinglePacket);
326 this.webRTCWrapper.close();
327 sendSessionTerminate(
328 Reason.FAILED_APPLICATION,
329 String.format(
330 "contents with names %s already exists",
331 Joiner.on(", ").join(modification.getNames())));
332 return;
333 }
334 final ContentAddition contentAddition =
335 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
336
337 final RtpContentMap outgoing = this.outgoingContentAdd;
338 final Set<ContentAddition.Summary> outgoingContentAddSummary =
339 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
340
341 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
342 if (isInitiator()) {
343 Log.d(
344 Config.LOGTAG,
345 id.getAccount().getJid().asBareJid()
346 + ": respond with tie break to matching content-add offer");
347 respondWithTieBreak(jinglePacket);
348 } else {
349 Log.d(
350 Config.LOGTAG,
351 id.getAccount().getJid().asBareJid()
352 + ": automatically accept matching content-add offer");
353 acceptContentAdd(contentAddition.summary, modification);
354 }
355 return;
356 }
357
358 // once we can display multiple video tracks we can be more loose with this condition
359 // theoretically it should also be fine to automatically accept audio only contents
360 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
361 Log.d(
362 Config.LOGTAG,
363 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
364 this.incomingContentAdd = modification;
365 respondOk(jinglePacket);
366 updateEndUserState();
367 } else {
368 respondOk(jinglePacket);
369 // TODO do we want to add a reason?
370 rejectContentAdd(modification);
371 }
372 }
373
374 private void receiveContentAccept(final JinglePacket jinglePacket) {
375 final RtpContentMap receivedContentAccept;
376 try {
377 receivedContentAccept = RtpContentMap.of(jinglePacket);
378 receivedContentAccept.requireContentDescriptions();
379 } catch (final RuntimeException e) {
380 Log.d(
381 Config.LOGTAG,
382 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
383 Throwables.getRootCause(e));
384 respondOk(jinglePacket);
385 webRTCWrapper.close();
386 sendSessionTerminate(Reason.of(e), e.getMessage());
387 return;
388 }
389
390 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
391 if (outgoingContentAdd == null) {
392 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
393 terminateWithOutOfOrder(jinglePacket);
394 return;
395 }
396 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
397 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
398 this.outgoingContentAdd = null;
399 respondOk(jinglePacket);
400 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
401 final ListenableFuture<RtpContentMap> future =
402 receiveRtpContentMap(
403 receivedContentAccept,
404 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
405 Futures.addCallback(
406 future,
407 new FutureCallback<>() {
408 @Override
409 public void onSuccess(final RtpContentMap result) {
410 receiveContentAccept(result);
411 }
412
413 @Override
414 public void onFailure(@NonNull final Throwable throwable) {
415 webRTCWrapper.close();
416 sendSessionTerminate(
417 Reason.ofThrowable(throwable), throwable.getMessage());
418 }
419 },
420 MoreExecutors.directExecutor());
421 } else {
422 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
423 terminateWithOutOfOrder(jinglePacket);
424 }
425 }
426
427 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
428 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
429 final RtpContentMap modifiedContentMap =
430 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
431
432 setRemoteContentMap(modifiedContentMap);
433
434 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
435
436 final org.webrtc.SessionDescription sdp =
437 new org.webrtc.SessionDescription(
438 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
439
440 try {
441 this.webRTCWrapper.setRemoteDescription(sdp).get();
442 } catch (final Exception e) {
443 final Throwable cause = Throwables.getRootCause(e);
444 Log.d(
445 Config.LOGTAG,
446 id.getAccount().getJid().asBareJid()
447 + ": unable to set remote description after receiving content-accept",
448 cause);
449 webRTCWrapper.close();
450 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
451 return;
452 }
453 Log.d(
454 Config.LOGTAG,
455 id.getAccount().getJid().asBareJid()
456 + ": remote has accepted content-add "
457 + ContentAddition.summary(receivedContentAccept));
458 processCandidates(receivedContentAccept.contents.entrySet());
459 updateEndUserState();
460 }
461
462 private void receiveContentModify(final JinglePacket jinglePacket) {
463 if (this.state != State.SESSION_ACCEPTED) {
464 terminateWithOutOfOrder(jinglePacket);
465 return;
466 }
467 final Map<String, Content.Senders> modification =
468 Maps.transformEntries(
469 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
470 final boolean isInitiator = isInitiator();
471 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
472 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
473 final Set<String> currentOutgoingMediaIds =
474 currentOutgoing == null
475 ? Collections.emptySet()
476 : currentOutgoing.contents.keySet();
477 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
478 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
479 respondOk(jinglePacket);
480 final RtpContentMap modifiedContentMap;
481 try {
482 modifiedContentMap =
483 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
484 } catch (final IllegalArgumentException e) {
485 webRTCWrapper.close();
486 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
487 return;
488 }
489 this.outgoingContentAdd = modifiedContentMap;
490 Log.d(
491 Config.LOGTAG,
492 id.account.getJid().asBareJid()
493 + ": processed content-modification for pending content-add");
494 } else if (remoteContentMap != null
495 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
496 respondOk(jinglePacket);
497 final RtpContentMap modifiedRemoteContentMap;
498 try {
499 modifiedRemoteContentMap =
500 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
501 } catch (final IllegalArgumentException e) {
502 webRTCWrapper.close();
503 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
504 return;
505 }
506 final SessionDescription offer;
507 try {
508 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
509 } catch (final IllegalArgumentException | NullPointerException e) {
510 Log.d(
511 Config.LOGTAG,
512 id.getAccount().getJid().asBareJid()
513 + ": unable convert offer from content-modify to SDP",
514 e);
515 webRTCWrapper.close();
516 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
517 return;
518 }
519 Log.d(
520 Config.LOGTAG,
521 id.account.getJid().asBareJid() + ": auto accepting content-modification");
522 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
523 } else {
524 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
525 respondWithItemNotFound(jinglePacket);
526 }
527 }
528
529 private void autoAcceptContentModify(
530 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
531 this.setRemoteContentMap(modifiedRemoteContentMap);
532 final org.webrtc.SessionDescription sdp =
533 new org.webrtc.SessionDescription(
534 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
535 try {
536 this.webRTCWrapper.setRemoteDescription(sdp).get();
537 // auto accept is only done when we already have tracks
538 final SessionDescription answer = setLocalSessionDescription();
539 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
540 modifyLocalContentMap(rtpContentMap);
541 // we do not need to send an answer but do we have to resend the candidates currently in
542 // SDP?
543 // resendCandidatesFromSdp(answer);
544 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
545 } catch (final Exception e) {
546 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
547 webRTCWrapper.close();
548 sendSessionTerminate(Reason.FAILED_APPLICATION);
549 }
550 }
551
552 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
553 final SessionDescription answer) {
554 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
555 new ImmutableMultimap.Builder<>();
556 for (final SessionDescription.Media media : answer.media) {
557 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
558 if (Strings.isNullOrEmpty(mid)) {
559 continue;
560 }
561 for (final String sdpCandidate : media.attributes.get("candidate")) {
562 final IceUdpTransportInfo.Candidate candidate =
563 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
564 if (candidate != null) {
565 candidateBuilder.put(mid, candidate);
566 }
567 }
568 }
569 return candidateBuilder.build();
570 }
571
572 private void receiveContentReject(final JinglePacket jinglePacket) {
573 final RtpContentMap receivedContentReject;
574 try {
575 receivedContentReject = RtpContentMap.of(jinglePacket);
576 } catch (final RuntimeException e) {
577 Log.d(
578 Config.LOGTAG,
579 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
580 Throwables.getRootCause(e));
581 respondOk(jinglePacket);
582 this.webRTCWrapper.close();
583 sendSessionTerminate(Reason.of(e), e.getMessage());
584 return;
585 }
586
587 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
588 if (outgoingContentAdd == null) {
589 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
590 terminateWithOutOfOrder(jinglePacket);
591 return;
592 }
593 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
594 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
595 this.outgoingContentAdd = null;
596 respondOk(jinglePacket);
597 Log.d(Config.LOGTAG, jinglePacket.toString());
598 receiveContentReject(ourSummary);
599 } else {
600 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
601 terminateWithOutOfOrder(jinglePacket);
602 }
603 }
604
605 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
606 try {
607 this.webRTCWrapper.removeTrack(Media.VIDEO);
608 final RtpContentMap localContentMap = customRollback();
609 modifyLocalContentMap(localContentMap);
610 } catch (final Exception e) {
611 final Throwable cause = Throwables.getRootCause(e);
612 Log.d(
613 Config.LOGTAG,
614 id.getAccount().getJid().asBareJid()
615 + ": unable to rollback local description after receiving content-reject",
616 cause);
617 webRTCWrapper.close();
618 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
619 return;
620 }
621 Log.d(
622 Config.LOGTAG,
623 id.getAccount().getJid().asBareJid()
624 + ": remote has rejected our content-add "
625 + summary);
626 }
627
628 private void receiveContentRemove(final JinglePacket jinglePacket) {
629 final RtpContentMap receivedContentRemove;
630 try {
631 receivedContentRemove = RtpContentMap.of(jinglePacket);
632 receivedContentRemove.requireContentDescriptions();
633 } catch (final RuntimeException e) {
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
637 Throwables.getRootCause(e));
638 respondOk(jinglePacket);
639 this.webRTCWrapper.close();
640 sendSessionTerminate(Reason.of(e), e.getMessage());
641 return;
642 }
643 respondOk(jinglePacket);
644 receiveContentRemove(receivedContentRemove);
645 }
646
647 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
648 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
649 final Set<ContentAddition.Summary> contentAddSummary =
650 incomingContentAdd == null
651 ? Collections.emptySet()
652 : ContentAddition.summary(incomingContentAdd);
653 final Set<ContentAddition.Summary> removeSummary =
654 ContentAddition.summary(receivedContentRemove);
655 if (contentAddSummary.equals(removeSummary)) {
656 this.incomingContentAdd = null;
657 updateEndUserState();
658 } else {
659 webRTCWrapper.close();
660 sendSessionTerminate(
661 Reason.FAILED_APPLICATION,
662 String.format(
663 "%s only supports %s as a means to retract a not yet accepted %s",
664 BuildConfig.APP_NAME,
665 JinglePacket.Action.CONTENT_REMOVE,
666 JinglePacket.Action.CONTENT_ADD));
667 }
668 }
669
670 public synchronized void retractContentAdd() {
671 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
672 if (outgoingContentAdd == null) {
673 throw new IllegalStateException("Not outgoing content add");
674 }
675 try {
676 webRTCWrapper.removeTrack(Media.VIDEO);
677 final RtpContentMap localContentMap = customRollback();
678 modifyLocalContentMap(localContentMap);
679 } catch (final Exception e) {
680 final Throwable cause = Throwables.getRootCause(e);
681 Log.d(
682 Config.LOGTAG,
683 id.getAccount().getJid().asBareJid()
684 + ": unable to rollback local description after trying to retract content-add",
685 cause);
686 webRTCWrapper.close();
687 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
688 return;
689 }
690 this.outgoingContentAdd = null;
691 final JinglePacket retract =
692 outgoingContentAdd
693 .toStub()
694 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
695 this.send(retract);
696 Log.d(
697 Config.LOGTAG,
698 id.getAccount().getJid()
699 + ": retract content-add "
700 + ContentAddition.summary(outgoingContentAdd));
701 }
702
703 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
704 final SessionDescription sdp = setLocalSessionDescription();
705 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
706 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
707 this.webRTCWrapper
708 .setRemoteDescription(
709 new org.webrtc.SessionDescription(
710 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
711 .get();
712 return localRtpContentMap;
713 }
714
715 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
716 final RtpContentMap currentRemote = getRemoteContentMap();
717 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
718 if (diff.isEmpty()) {
719 throw new IllegalStateException(
720 "Unexpected rollback condition. No difference between local and remote");
721 }
722 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
723 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
724 final RtpContentMap nextRemote =
725 currentRemote.addContent(
726 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
727 return SessionDescription.of(nextRemote, isResponder());
728 }
729 throw new IllegalStateException(
730 "Unexpected rollback condition. Senders were not uniformly none");
731 }
732
733 public synchronized void acceptContentAdd(
734 @NonNull final Set<ContentAddition.Summary> contentAddition) {
735 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
736 if (incomingContentAdd == null) {
737 throw new IllegalStateException("No incoming content add");
738 }
739
740 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
741 this.incomingContentAdd = null;
742 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
743 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
744 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
745 Log.d(
746 Config.LOGTAG,
747 "content addition is receive only. we want to upgrade to 'both'");
748 final RtpContentMap modifiedSenders =
749 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
750 final JinglePacket proposedContentModification =
751 modifiedSenders
752 .toStub()
753 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
754 proposedContentModification.setTo(id.with);
755 xmppConnectionService.sendIqPacket(
756 id.account,
757 proposedContentModification,
758 (account, response) -> {
759 if (response.getType() == IqPacket.TYPE.RESULT) {
760 Log.d(
761 Config.LOGTAG,
762 id.account.getJid().asBareJid()
763 + ": remote has accepted our upgrade to senders=both");
764 acceptContentAdd(
765 ContentAddition.summary(modifiedSenders), modifiedSenders);
766 } else {
767 Log.d(
768 Config.LOGTAG,
769 id.account.getJid().asBareJid()
770 + ": remote has rejected our upgrade to senders=both");
771 acceptContentAdd(contentAddition, incomingContentAdd);
772 }
773 });
774 } else {
775 acceptContentAdd(contentAddition, incomingContentAdd);
776 }
777 } else {
778 throw new IllegalStateException(
779 "Accepted content add does not match pending content-add");
780 }
781 }
782
783 private void acceptContentAdd(
784 @NonNull final Set<ContentAddition.Summary> contentAddition,
785 final RtpContentMap incomingContentAdd) {
786 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
787 final RtpContentMap modifiedContentMap =
788 getRemoteContentMap().addContent(incomingContentAdd, setup);
789 this.setRemoteContentMap(modifiedContentMap);
790
791 final SessionDescription offer;
792 try {
793 offer = SessionDescription.of(modifiedContentMap, isResponder());
794 } catch (final IllegalArgumentException | NullPointerException e) {
795 Log.d(
796 Config.LOGTAG,
797 id.getAccount().getJid().asBareJid()
798 + ": unable convert offer from content-add to SDP",
799 e);
800 webRTCWrapper.close();
801 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
802 return;
803 }
804 this.incomingContentAdd = null;
805 acceptContentAdd(contentAddition, offer);
806 }
807
808 private void acceptContentAdd(
809 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
810 final org.webrtc.SessionDescription sdp =
811 new org.webrtc.SessionDescription(
812 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
813 try {
814 this.webRTCWrapper.setRemoteDescription(sdp).get();
815
816 // TODO add tracks for 'media' where contentAddition.senders matches
817
818 // TODO if senders.sending(isInitiator())
819
820 this.webRTCWrapper.addTrack(Media.VIDEO);
821
822 // TODO add additional transceivers for recv only cases
823
824 final SessionDescription answer = setLocalSessionDescription();
825 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
826
827 final RtpContentMap contentAcceptMap =
828 rtpContentMap.toContentModification(
829 Collections2.transform(contentAddition, ca -> ca.name));
830
831 Log.d(
832 Config.LOGTAG,
833 id.getAccount().getJid().asBareJid()
834 + ": sending content-accept "
835 + ContentAddition.summary(contentAcceptMap));
836
837 addIceCandidatesFromBlackLog();
838
839 modifyLocalContentMap(rtpContentMap);
840 final ListenableFuture<RtpContentMap> future =
841 prepareOutgoingContentMap(contentAcceptMap);
842 Futures.addCallback(
843 future,
844 new FutureCallback<>() {
845 @Override
846 public void onSuccess(final RtpContentMap rtpContentMap) {
847 sendContentAccept(rtpContentMap);
848 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
849 }
850
851 @Override
852 public void onFailure(@NonNull final Throwable throwable) {
853 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
854 }
855 },
856 MoreExecutors.directExecutor());
857 } catch (final Exception e) {
858 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
859 webRTCWrapper.close();
860 sendSessionTerminate(Reason.FAILED_APPLICATION);
861 }
862 }
863
864 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
865 final JinglePacket jinglePacket =
866 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
867 send(jinglePacket);
868 }
869
870 public synchronized void rejectContentAdd() {
871 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
872 if (incomingContentAdd == null) {
873 throw new IllegalStateException("No incoming content add");
874 }
875 this.incomingContentAdd = null;
876 updateEndUserState();
877 rejectContentAdd(incomingContentAdd);
878 }
879
880 private void rejectContentAdd(final RtpContentMap contentMap) {
881 final JinglePacket jinglePacket =
882 contentMap
883 .toStub()
884 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
885 Log.d(
886 Config.LOGTAG,
887 id.getAccount().getJid().asBareJid()
888 + ": rejecting content "
889 + ContentAddition.summary(contentMap));
890 send(jinglePacket);
891 }
892
893 private boolean checkForIceRestart(
894 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
895 final RtpContentMap existing = getRemoteContentMap();
896 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
897 final IceUdpTransportInfo.Credentials newCredentials;
898 try {
899 existingCredentials = existing.getCredentials();
900 newCredentials = rtpContentMap.getDistinctCredentials();
901 } catch (final IllegalStateException e) {
902 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
903 return false;
904 }
905 if (existingCredentials.contains(newCredentials)) {
906 return false;
907 }
908 // TODO an alternative approach is to check if we already got an iq result to our
909 // ICE-restart
910 // and if that's the case we are seeing an answer.
911 // This might be more spec compliant but also more error prone potentially
912 final boolean isSignalStateStable =
913 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
914 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
915 final boolean isOffer = rtpContentMap.emptyCandidates();
916 final RtpContentMap restartContentMap;
917 try {
918 if (isOffer) {
919 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
920 restartContentMap =
921 existing.modifiedCredentials(
922 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
923 } else {
924 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
925 Log.d(
926 Config.LOGTAG,
927 "received confirmation of ICE restart"
928 + newCredentials
929 + " peer_setup="
930 + setup);
931 // DTLS setup attribute needs to be rewritten to reflect current peer state
932 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
933 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
934 }
935 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
936 return isOffer;
937 } else {
938 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
939 respondWithTieBreak(jinglePacket);
940 return true;
941 }
942 } catch (final Exception exception) {
943 respondOk(jinglePacket);
944 final Throwable rootCause = Throwables.getRootCause(exception);
945 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
946 // If this happens a termination is already in progress
947 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
948 return true;
949 }
950 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
951 webRTCWrapper.close();
952 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
953 return true;
954 }
955 }
956
957 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
958 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
959 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
960 throw new IllegalStateException("Invalid peer setup");
961 }
962 return peerSetup;
963 }
964
965 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
966 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
967 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
968 }
969 this.peerDtlsSetup = setup;
970 }
971
972 private boolean applyIceRestart(
973 final JinglePacket jinglePacket,
974 final RtpContentMap restartContentMap,
975 final boolean isOffer)
976 throws ExecutionException, InterruptedException {
977 final SessionDescription sessionDescription =
978 SessionDescription.of(restartContentMap, isResponder());
979 final org.webrtc.SessionDescription.Type type =
980 isOffer
981 ? org.webrtc.SessionDescription.Type.OFFER
982 : org.webrtc.SessionDescription.Type.ANSWER;
983 org.webrtc.SessionDescription sdp =
984 new org.webrtc.SessionDescription(type, sessionDescription.toString());
985 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
986 if (isInitiator()) {
987 // We ignore the offer and respond with tie-break. This will clause the responder
988 // not to apply the content map
989 return false;
990 }
991 }
992 webRTCWrapper.setRemoteDescription(sdp).get();
993 setRemoteContentMap(restartContentMap);
994 if (isOffer) {
995 final SessionDescription localSessionDescription = setLocalSessionDescription();
996 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
997 // We need to respond OK before sending any candidates
998 respondOk(jinglePacket);
999 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1000 } else {
1001 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1002 }
1003 return true;
1004 }
1005
1006 private void processCandidates(
1007 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> contents) {
1008 for (final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content : contents) {
1009 processCandidate(content);
1010 }
1011 }
1012
1013 private void processCandidate(
1014 final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content) {
1015 final RtpContentMap rtpContentMap = getRemoteContentMap();
1016 final List<String> indices = toIdentificationTags(rtpContentMap);
1017 final String sdpMid = content.getKey(); // aka content name
1018 final IceUdpTransportInfo transport = content.getValue().transport;
1019 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1020
1021 // TODO check that credentials remained the same
1022
1023 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1024 final String sdp;
1025 try {
1026 sdp = candidate.toSdpAttribute(credentials.ufrag);
1027 } catch (final IllegalArgumentException e) {
1028 Log.d(
1029 Config.LOGTAG,
1030 id.account.getJid().asBareJid()
1031 + ": ignoring invalid ICE candidate "
1032 + e.getMessage());
1033 continue;
1034 }
1035 final int mLineIndex = indices.indexOf(sdpMid);
1036 if (mLineIndex < 0) {
1037 Log.w(
1038 Config.LOGTAG,
1039 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1040 }
1041 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1042 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1043 this.webRTCWrapper.addIceCandidate(iceCandidate);
1044 }
1045 }
1046
1047 private RtpContentMap getRemoteContentMap() {
1048 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1049 }
1050
1051 private RtpContentMap getLocalContentMap() {
1052 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1053 }
1054
1055 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1056 final Group originalGroup = rtpContentMap.group;
1057 final List<String> identificationTags =
1058 originalGroup == null
1059 ? rtpContentMap.getNames()
1060 : originalGroup.getIdentificationTags();
1061 if (identificationTags.size() == 0) {
1062 Log.w(
1063 Config.LOGTAG,
1064 id.account.getJid().asBareJid()
1065 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1066 }
1067 return identificationTags;
1068 }
1069
1070 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1071 final JinglePacket jinglePacket, final boolean expectVerification) {
1072 try {
1073 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1074 } catch (final Exception e) {
1075 return Futures.immediateFailedFuture(e);
1076 }
1077 }
1078
1079 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1080 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1081 Log.d(
1082 Config.LOGTAG,
1083 "receiveRtpContentMap("
1084 + receivedContentMap.getClass().getSimpleName()
1085 + ",expectVerification="
1086 + expectVerification
1087 + ")");
1088 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1089 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1090 id.account
1091 .getAxolotlService()
1092 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1093 return Futures.transform(
1094 future,
1095 omemoVerifiedPayload -> {
1096 // TODO test if an exception here triggers a correct abort
1097 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1098 Log.d(
1099 Config.LOGTAG,
1100 id.account.getJid().asBareJid()
1101 + ": received verifiable DTLS fingerprint via "
1102 + omemoVerification);
1103 return omemoVerifiedPayload.getPayload();
1104 },
1105 MoreExecutors.directExecutor());
1106 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1107 return Futures.immediateFailedFuture(
1108 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1109 } else {
1110 return Futures.immediateFuture(receivedContentMap);
1111 }
1112 }
1113
1114 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1115 if (isInitiator()) {
1116 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1117 return;
1118 }
1119 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1120 Futures.addCallback(
1121 future,
1122 new FutureCallback<>() {
1123 @Override
1124 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1125 receiveSessionInitiate(jinglePacket, rtpContentMap);
1126 }
1127
1128 @Override
1129 public void onFailure(@NonNull final Throwable throwable) {
1130 respondOk(jinglePacket);
1131 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1132 }
1133 },
1134 MoreExecutors.directExecutor());
1135 }
1136
1137 private void receiveSessionInitiate(
1138 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1139 try {
1140 contentMap.requireContentDescriptions();
1141 contentMap.requireDTLSFingerprint(true);
1142 } catch (final RuntimeException e) {
1143 Log.d(
1144 Config.LOGTAG,
1145 id.account.getJid().asBareJid() + ": improperly formatted contents",
1146 Throwables.getRootCause(e));
1147 respondOk(jinglePacket);
1148 sendSessionTerminate(Reason.of(e), e.getMessage());
1149 return;
1150 }
1151 Log.d(
1152 Config.LOGTAG,
1153 "processing session-init with " + contentMap.contents.size() + " contents");
1154 final State target;
1155 if (this.state == State.PROCEED) {
1156 Preconditions.checkState(
1157 proposedMedia != null && proposedMedia.size() > 0,
1158 "proposed media must be set when processing pre-approved session-initiate");
1159 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1160 sendSessionTerminate(
1161 Reason.SECURITY_ERROR,
1162 String.format(
1163 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1164 this.proposedMedia, contentMap.getMedia()));
1165 return;
1166 }
1167 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1168 } else {
1169 target = State.SESSION_INITIALIZED;
1170 setProposedMedia(contentMap.getMedia());
1171 }
1172 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1173 respondOk(jinglePacket);
1174 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1175 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1176 Log.d(
1177 Config.LOGTAG,
1178 id.account.getJid().asBareJid()
1179 + ": automatically accepting session-initiate");
1180 sendSessionAccept();
1181 } else {
1182 Log.d(
1183 Config.LOGTAG,
1184 id.account.getJid().asBareJid()
1185 + ": received not pre-approved session-initiate. start ringing");
1186 startRinging();
1187 }
1188 } else {
1189 Log.d(
1190 Config.LOGTAG,
1191 String.format(
1192 "%s: received session-initiate while in state %s",
1193 id.account.getJid().asBareJid(), state));
1194 terminateWithOutOfOrder(jinglePacket);
1195 }
1196 }
1197
1198 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1199 if (isResponder()) {
1200 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1201 return;
1202 }
1203 final ListenableFuture<RtpContentMap> future =
1204 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1205 Futures.addCallback(
1206 future,
1207 new FutureCallback<>() {
1208 @Override
1209 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1210 receiveSessionAccept(jinglePacket, rtpContentMap);
1211 }
1212
1213 @Override
1214 public void onFailure(@NonNull final Throwable throwable) {
1215 respondOk(jinglePacket);
1216 Log.d(
1217 Config.LOGTAG,
1218 id.account.getJid().asBareJid()
1219 + ": improperly formatted contents in session-accept",
1220 throwable);
1221 webRTCWrapper.close();
1222 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1223 }
1224 },
1225 MoreExecutors.directExecutor());
1226 }
1227
1228 private void receiveSessionAccept(
1229 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1230 try {
1231 contentMap.requireContentDescriptions();
1232 contentMap.requireDTLSFingerprint();
1233 } catch (final RuntimeException e) {
1234 respondOk(jinglePacket);
1235 Log.d(
1236 Config.LOGTAG,
1237 id.account.getJid().asBareJid()
1238 + ": improperly formatted contents in session-accept",
1239 e);
1240 webRTCWrapper.close();
1241 sendSessionTerminate(Reason.of(e), e.getMessage());
1242 return;
1243 }
1244 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1245 if (!initiatorMedia.equals(contentMap.getMedia())) {
1246 sendSessionTerminate(
1247 Reason.SECURITY_ERROR,
1248 String.format(
1249 "Your session-included included media %s but our session-initiate was %s",
1250 this.proposedMedia, contentMap.getMedia()));
1251 return;
1252 }
1253 Log.d(
1254 Config.LOGTAG,
1255 "processing session-accept with " + contentMap.contents.size() + " contents");
1256 if (transition(State.SESSION_ACCEPTED)) {
1257 respondOk(jinglePacket);
1258 receiveSessionAccept(contentMap);
1259 } else {
1260 Log.d(
1261 Config.LOGTAG,
1262 String.format(
1263 "%s: received session-accept while in state %s",
1264 id.account.getJid().asBareJid(), state));
1265 respondOk(jinglePacket);
1266 }
1267 }
1268
1269 private void receiveSessionAccept(final RtpContentMap contentMap) {
1270 this.responderRtpContentMap = contentMap;
1271 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1272 final SessionDescription sessionDescription;
1273 try {
1274 sessionDescription = SessionDescription.of(contentMap, false);
1275 } catch (final IllegalArgumentException | NullPointerException e) {
1276 Log.d(
1277 Config.LOGTAG,
1278 id.account.getJid().asBareJid()
1279 + ": unable convert offer from session-accept to SDP",
1280 e);
1281 webRTCWrapper.close();
1282 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1283 return;
1284 }
1285 final org.webrtc.SessionDescription answer =
1286 new org.webrtc.SessionDescription(
1287 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1288 try {
1289 this.webRTCWrapper.setRemoteDescription(answer).get();
1290 } catch (final Exception e) {
1291 Log.d(
1292 Config.LOGTAG,
1293 id.account.getJid().asBareJid()
1294 + ": unable to set remote description after receiving session-accept",
1295 Throwables.getRootCause(e));
1296 webRTCWrapper.close();
1297 sendSessionTerminate(
1298 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1299 return;
1300 }
1301 processCandidates(contentMap.contents.entrySet());
1302 }
1303
1304 private void sendSessionAccept() {
1305 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1306 if (rtpContentMap == null) {
1307 throw new IllegalStateException("initiator RTP Content Map has not been set");
1308 }
1309 final SessionDescription offer;
1310 try {
1311 offer = SessionDescription.of(rtpContentMap, true);
1312 } catch (final IllegalArgumentException | NullPointerException e) {
1313 Log.d(
1314 Config.LOGTAG,
1315 id.account.getJid().asBareJid()
1316 + ": unable convert offer from session-initiate to SDP",
1317 e);
1318 webRTCWrapper.close();
1319 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1320 return;
1321 }
1322 sendSessionAccept(rtpContentMap.getMedia(), offer);
1323 }
1324
1325 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1326 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1327 }
1328
1329 private synchronized void sendSessionAccept(
1330 final Set<Media> media,
1331 final SessionDescription offer,
1332 final List<PeerConnection.IceServer> iceServers) {
1333 if (isTerminated()) {
1334 Log.w(
1335 Config.LOGTAG,
1336 id.account.getJid().asBareJid()
1337 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1338 return;
1339 }
1340 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1341 try {
1342 setupWebRTC(media, iceServers, !includeCandidates);
1343 } catch (final WebRTCWrapper.InitializationException e) {
1344 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1345 webRTCWrapper.close();
1346 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1347 return;
1348 }
1349 final org.webrtc.SessionDescription sdp =
1350 new org.webrtc.SessionDescription(
1351 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1352 try {
1353 this.webRTCWrapper.setRemoteDescription(sdp).get();
1354 addIceCandidatesFromBlackLog();
1355 org.webrtc.SessionDescription webRTCSessionDescription =
1356 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1357 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1358 } catch (final Exception e) {
1359 failureToAcceptSession(e);
1360 }
1361 }
1362
1363 private void failureToAcceptSession(final Throwable throwable) {
1364 if (isTerminated()) {
1365 return;
1366 }
1367 final Throwable rootCause = Throwables.getRootCause(throwable);
1368 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1369 webRTCWrapper.close();
1370 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1371 }
1372
1373 private void failureToPerformAction(
1374 final JinglePacket.Action action, final Throwable throwable) {
1375 if (isTerminated()) {
1376 return;
1377 }
1378 final Throwable rootCause = Throwables.getRootCause(throwable);
1379 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1380 webRTCWrapper.close();
1381 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1382 }
1383
1384 private void addIceCandidatesFromBlackLog() {
1385 Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> foo;
1386 while ((foo = this.pendingIceCandidates.poll()) != null) {
1387 processCandidate(foo);
1388 Log.d(
1389 Config.LOGTAG,
1390 id.account.getJid().asBareJid() + ": added candidate from back log");
1391 }
1392 }
1393
1394 private void prepareSessionAccept(
1395 final org.webrtc.SessionDescription webRTCSessionDescription,
1396 final boolean includeCandidates) {
1397 final SessionDescription sessionDescription =
1398 SessionDescription.parse(webRTCSessionDescription.description);
1399 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1400 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1401 if (includeCandidates) {
1402 candidates = parseCandidates(sessionDescription);
1403 } else {
1404 candidates = ImmutableMultimap.of();
1405 }
1406 this.responderRtpContentMap = respondingRtpContentMap;
1407 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1408 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1409 prepareOutgoingContentMap(respondingRtpContentMap);
1410 Futures.addCallback(
1411 outgoingContentMapFuture,
1412 new FutureCallback<>() {
1413 @Override
1414 public void onSuccess(final RtpContentMap outgoingContentMap) {
1415 if (includeCandidates) {
1416 Log.d(
1417 Config.LOGTAG,
1418 "including "
1419 + candidates.size()
1420 + " candidates in session accept");
1421 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1422 } else {
1423 sendSessionAccept(outgoingContentMap);
1424 }
1425 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1426 }
1427
1428 @Override
1429 public void onFailure(@NonNull Throwable throwable) {
1430 failureToAcceptSession(throwable);
1431 }
1432 },
1433 MoreExecutors.directExecutor());
1434 }
1435
1436 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1437 if (isTerminated()) {
1438 Log.w(
1439 Config.LOGTAG,
1440 id.account.getJid().asBareJid()
1441 + ": preparing session accept was too slow. already terminated. nothing to do.");
1442 return;
1443 }
1444 transitionOrThrow(State.SESSION_ACCEPTED);
1445 final JinglePacket sessionAccept =
1446 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1447 send(sessionAccept);
1448 }
1449
1450 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1451 final RtpContentMap rtpContentMap) {
1452 if (this.omemoVerification.hasDeviceId()) {
1453 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1454 verifiedPayloadFuture =
1455 id.account
1456 .getAxolotlService()
1457 .encrypt(
1458 rtpContentMap,
1459 id.with,
1460 omemoVerification.getDeviceId());
1461 return Futures.transform(
1462 verifiedPayloadFuture,
1463 verifiedPayload -> {
1464 omemoVerification.setOrEnsureEqual(verifiedPayload);
1465 return verifiedPayload.getPayload();
1466 },
1467 MoreExecutors.directExecutor());
1468 } else {
1469 return Futures.immediateFuture(rtpContentMap);
1470 }
1471 }
1472
1473 synchronized void deliveryMessage(
1474 final Jid from,
1475 final Element message,
1476 final String serverMessageId,
1477 final long timestamp) {
1478 Log.d(
1479 Config.LOGTAG,
1480 id.account.getJid().asBareJid()
1481 + ": delivered message to JingleRtpConnection "
1482 + message);
1483 switch (message.getName()) {
1484 case "propose" -> receivePropose(
1485 from, Propose.upgrade(message), serverMessageId, timestamp);
1486 case "proceed" -> receiveProceed(
1487 from, Proceed.upgrade(message), serverMessageId, timestamp);
1488 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1489 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1490 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1491 }
1492 }
1493
1494 void deliverFailedProceed(final String message) {
1495 Log.d(
1496 Config.LOGTAG,
1497 id.account.getJid().asBareJid()
1498 + ": receive message error for proceed message ("
1499 + Strings.nullToEmpty(message)
1500 + ")");
1501 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1502 webRTCWrapper.close();
1503 Log.d(
1504 Config.LOGTAG,
1505 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1506 this.finish();
1507 }
1508 }
1509
1510 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1511 final boolean originatedFromMyself =
1512 from.asBareJid().equals(id.account.getJid().asBareJid());
1513 if (originatedFromMyself) {
1514 if (transition(State.ACCEPTED)) {
1515 acceptedOnOtherDevice(serverMsgId, timestamp);
1516 } else {
1517 Log.d(
1518 Config.LOGTAG,
1519 id.account.getJid().asBareJid()
1520 + ": unable to transition to accept because already in state="
1521 + this.state);
1522 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1523 }
1524 } else {
1525 Log.d(
1526 Config.LOGTAG,
1527 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1528 }
1529 }
1530
1531 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1532 if (serverMsgId != null) {
1533 this.message.setServerMsgId(serverMsgId);
1534 }
1535 this.message.setTime(timestamp);
1536 this.message.setCarbon(true); // indicate that call was accepted on other device
1537 this.writeLogMessageSuccess(0);
1538 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1539 this.finish();
1540 }
1541
1542 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1543 final boolean originatedFromMyself =
1544 from.asBareJid().equals(id.account.getJid().asBareJid());
1545 // reject from another one of my clients
1546 if (originatedFromMyself) {
1547 receiveRejectFromMyself(serverMsgId, timestamp);
1548 } else if (isInitiator()) {
1549 if (from.equals(id.with)) {
1550 receiveRejectFromResponder();
1551 } else {
1552 Log.d(
1553 Config.LOGTAG,
1554 id.account.getJid()
1555 + ": ignoring reject from "
1556 + from
1557 + " for session with "
1558 + id.with);
1559 }
1560 } else {
1561 Log.d(
1562 Config.LOGTAG,
1563 id.account.getJid()
1564 + ": ignoring reject from "
1565 + from
1566 + " for session with "
1567 + id.with);
1568 }
1569 }
1570
1571 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1572 if (transition(State.REJECTED)) {
1573 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1574 this.finish();
1575 if (serverMsgId != null) {
1576 this.message.setServerMsgId(serverMsgId);
1577 }
1578 this.message.setTime(timestamp);
1579 this.message.setCarbon(true); // indicate that call was rejected on other device
1580 writeLogMessageMissed();
1581 } else {
1582 Log.d(
1583 Config.LOGTAG,
1584 "not able to transition into REJECTED because already in " + this.state);
1585 }
1586 }
1587
1588 private void receiveRejectFromResponder() {
1589 if (isInState(State.PROCEED)) {
1590 Log.d(
1591 Config.LOGTAG,
1592 id.account.getJid()
1593 + ": received reject while still in proceed. callee reconsidered");
1594 closeTransitionLogFinish(State.REJECTED_RACED);
1595 return;
1596 }
1597 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1598 Log.d(
1599 Config.LOGTAG,
1600 id.account.getJid()
1601 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1602 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1603 return;
1604 }
1605 Log.d(
1606 Config.LOGTAG,
1607 id.account.getJid()
1608 + ": ignoring reject from responder because already in state "
1609 + this.state);
1610 }
1611
1612 private void receivePropose(
1613 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1614 final boolean originatedFromMyself =
1615 from.asBareJid().equals(id.account.getJid().asBareJid());
1616 if (originatedFromMyself) {
1617 Log.d(
1618 Config.LOGTAG,
1619 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1620 } else if (transition(
1621 State.PROPOSED,
1622 () -> {
1623 final Collection<RtpDescription> descriptions =
1624 Collections2.transform(
1625 Collections2.filter(
1626 propose.getDescriptions(),
1627 d -> d instanceof RtpDescription),
1628 input -> (RtpDescription) input);
1629 final Collection<Media> media =
1630 Collections2.transform(descriptions, RtpDescription::getMedia);
1631 Preconditions.checkState(
1632 !media.contains(Media.UNKNOWN),
1633 "RTP descriptions contain unknown media");
1634 Log.d(
1635 Config.LOGTAG,
1636 id.account.getJid().asBareJid()
1637 + ": received session proposal from "
1638 + from
1639 + " for "
1640 + media);
1641 this.setProposedMedia(Sets.newHashSet(media));
1642 })) {
1643 if (serverMsgId != null) {
1644 this.message.setServerMsgId(serverMsgId);
1645 }
1646 this.message.setTime(timestamp);
1647 startRinging();
1648 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1649 sendJingleMessage("ringing");
1650 }
1651 } else {
1652 Log.d(
1653 Config.LOGTAG,
1654 id.account.getJid()
1655 + ": ignoring session proposal because already in "
1656 + state);
1657 }
1658 }
1659
1660 private void startRinging() {
1661 this.callIntegration.setRinging();
1662 Log.d(
1663 Config.LOGTAG,
1664 id.account.getJid().asBareJid()
1665 + ": received call from "
1666 + id.with
1667 + ". start ringing");
1668 ringingTimeoutFuture =
1669 jingleConnectionManager.schedule(
1670 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1671 if (CallIntegration.selfManaged()) {
1672 return;
1673 }
1674 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1675 }
1676
1677 private synchronized void ringingTimeout() {
1678 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1679 switch (this.state) {
1680 case PROPOSED -> {
1681 message.markUnread();
1682 rejectCallFromProposed();
1683 }
1684 case SESSION_INITIALIZED -> {
1685 message.markUnread();
1686 rejectCallFromSessionInitiate();
1687 }
1688 }
1689 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1690 }
1691
1692 private void cancelRingingTimeout() {
1693 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1694 if (future != null && !future.isCancelled()) {
1695 future.cancel(false);
1696 }
1697 }
1698
1699 private void receiveProceed(
1700 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1701 final Set<Media> media =
1702 Preconditions.checkNotNull(
1703 this.proposedMedia, "Proposed media has to be set before handling proceed");
1704 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1705 if (from.equals(id.with)) {
1706 if (isInitiator()) {
1707 if (transition(State.PROCEED)) {
1708 if (serverMsgId != null) {
1709 this.message.setServerMsgId(serverMsgId);
1710 }
1711 this.message.setTime(timestamp);
1712 final Integer remoteDeviceId = proceed.getDeviceId();
1713 if (isOmemoEnabled()) {
1714 this.omemoVerification.setDeviceId(remoteDeviceId);
1715 } else {
1716 if (remoteDeviceId != null) {
1717 Log.d(
1718 Config.LOGTAG,
1719 id.account.getJid().asBareJid()
1720 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1721 }
1722 this.omemoVerification.setDeviceId(null);
1723 }
1724 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1725 } else {
1726 Log.d(
1727 Config.LOGTAG,
1728 String.format(
1729 "%s: ignoring proceed because already in %s",
1730 id.account.getJid().asBareJid(), this.state));
1731 }
1732 } else {
1733 Log.d(
1734 Config.LOGTAG,
1735 String.format(
1736 "%s: ignoring proceed because we were not initializing",
1737 id.account.getJid().asBareJid()));
1738 }
1739 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1740 if (transition(State.ACCEPTED)) {
1741 Log.d(
1742 Config.LOGTAG,
1743 id.account.getJid().asBareJid()
1744 + ": moved session with "
1745 + id.with
1746 + " into state accepted after received carbon copied proceed");
1747 acceptedOnOtherDevice(serverMsgId, timestamp);
1748 }
1749 } else {
1750 Log.d(
1751 Config.LOGTAG,
1752 String.format(
1753 "%s: ignoring proceed from %s. was expected from %s",
1754 id.account.getJid().asBareJid(), from, id.with));
1755 }
1756 }
1757
1758 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1759 if (from.equals(id.with)) {
1760 final State target =
1761 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1762 if (transition(target)) {
1763 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1764 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1765 Log.d(
1766 Config.LOGTAG,
1767 id.account.getJid().asBareJid()
1768 + ": session with "
1769 + id.with
1770 + " has been retracted (serverMsgId="
1771 + serverMsgId
1772 + ")");
1773 if (serverMsgId != null) {
1774 this.message.setServerMsgId(serverMsgId);
1775 }
1776 this.message.setTime(timestamp);
1777 if (target == State.RETRACTED) {
1778 this.message.markUnread();
1779 }
1780 writeLogMessageMissed();
1781 finish();
1782 } else {
1783 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1784 }
1785 } else {
1786 // TODO parse retract from self
1787 Log.d(
1788 Config.LOGTAG,
1789 id.account.getJid().asBareJid()
1790 + ": received retract from "
1791 + from
1792 + ". expected retract from"
1793 + id.with
1794 + ". ignoring");
1795 }
1796 }
1797
1798 public void sendSessionInitiate() {
1799 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1800 }
1801
1802 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1803 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1804 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1805 }
1806
1807 private synchronized void sendSessionInitiate(
1808 final Set<Media> media,
1809 final State targetState,
1810 final List<PeerConnection.IceServer> iceServers) {
1811 if (isTerminated()) {
1812 Log.w(
1813 Config.LOGTAG,
1814 id.account.getJid().asBareJid()
1815 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1816 return;
1817 }
1818 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1819 try {
1820 setupWebRTC(media, iceServers, !includeCandidates);
1821 } catch (final WebRTCWrapper.InitializationException e) {
1822 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1823 webRTCWrapper.close();
1824 sendRetract(Reason.ofThrowable(e));
1825 return;
1826 }
1827 try {
1828 org.webrtc.SessionDescription webRTCSessionDescription =
1829 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1830 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1831 } catch (final Exception e) {
1832 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1833 // exceptions
1834 failureToInitiateSession(e, targetState);
1835 }
1836 }
1837
1838 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1839 if (isTerminated()) {
1840 return;
1841 }
1842 Log.d(
1843 Config.LOGTAG,
1844 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1845 Throwables.getRootCause(throwable));
1846 webRTCWrapper.close();
1847 final Reason reason = Reason.ofThrowable(throwable);
1848 if (isInState(targetState)) {
1849 sendSessionTerminate(reason, throwable.getMessage());
1850 } else {
1851 sendRetract(reason);
1852 }
1853 }
1854
1855 private void sendRetract(final Reason reason) {
1856 // TODO embed reason into retract
1857 sendJingleMessage("retract", id.with.asBareJid());
1858 transitionOrThrow(reasonToState(reason));
1859 this.finish();
1860 }
1861
1862 private void prepareSessionInitiate(
1863 final org.webrtc.SessionDescription webRTCSessionDescription,
1864 final boolean includeCandidates,
1865 final State targetState) {
1866 final SessionDescription sessionDescription =
1867 SessionDescription.parse(webRTCSessionDescription.description);
1868 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1869 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1870 if (includeCandidates) {
1871 candidates = parseCandidates(sessionDescription);
1872 } else {
1873 candidates = ImmutableMultimap.of();
1874 }
1875 this.initiatorRtpContentMap = rtpContentMap;
1876 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1877 encryptSessionInitiate(rtpContentMap);
1878 Futures.addCallback(
1879 outgoingContentMapFuture,
1880 new FutureCallback<>() {
1881 @Override
1882 public void onSuccess(final RtpContentMap outgoingContentMap) {
1883 if (includeCandidates) {
1884 Log.d(
1885 Config.LOGTAG,
1886 "including "
1887 + candidates.size()
1888 + " candidates in session initiate");
1889 sendSessionInitiate(
1890 outgoingContentMap.withCandidates(candidates), targetState);
1891 } else {
1892 sendSessionInitiate(outgoingContentMap, targetState);
1893 }
1894 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1895 }
1896
1897 @Override
1898 public void onFailure(@NonNull final Throwable throwable) {
1899 failureToInitiateSession(throwable, targetState);
1900 }
1901 },
1902 MoreExecutors.directExecutor());
1903 }
1904
1905 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1906 if (isTerminated()) {
1907 Log.w(
1908 Config.LOGTAG,
1909 id.account.getJid().asBareJid()
1910 + ": preparing session was too slow. already terminated. nothing to do.");
1911 return;
1912 }
1913 this.transitionOrThrow(targetState);
1914 final JinglePacket sessionInitiate =
1915 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1916 send(sessionInitiate);
1917 }
1918
1919 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1920 final RtpContentMap rtpContentMap) {
1921 if (this.omemoVerification.hasDeviceId()) {
1922 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1923 verifiedPayloadFuture =
1924 id.account
1925 .getAxolotlService()
1926 .encrypt(
1927 rtpContentMap,
1928 id.with,
1929 omemoVerification.getDeviceId());
1930 final ListenableFuture<RtpContentMap> future =
1931 Futures.transform(
1932 verifiedPayloadFuture,
1933 verifiedPayload -> {
1934 omemoVerification.setSessionFingerprint(
1935 verifiedPayload.getFingerprint());
1936 return verifiedPayload.getPayload();
1937 },
1938 MoreExecutors.directExecutor());
1939 if (Config.REQUIRE_RTP_VERIFICATION) {
1940 return future;
1941 }
1942 return Futures.catching(
1943 future,
1944 CryptoFailedException.class,
1945 e -> {
1946 Log.w(
1947 Config.LOGTAG,
1948 id.account.getJid().asBareJid()
1949 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1950 e);
1951 return rtpContentMap;
1952 },
1953 MoreExecutors.directExecutor());
1954 } else {
1955 return Futures.immediateFuture(rtpContentMap);
1956 }
1957 }
1958
1959 protected void sendSessionTerminate(final Reason reason) {
1960 sendSessionTerminate(reason, null);
1961 }
1962
1963
1964 protected void sendSessionTerminate(final Reason reason, final String text) {
1965 sendSessionTerminate(reason,text, this::writeLogMessage);
1966 }
1967
1968
1969 private void sendTransportInfo(
1970 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1971 final RtpContentMap transportInfo;
1972 try {
1973 final RtpContentMap rtpContentMap =
1974 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1975 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1976 } catch (final Exception e) {
1977 Log.d(
1978 Config.LOGTAG,
1979 id.account.getJid().asBareJid()
1980 + ": unable to prepare transport-info from candidate for content="
1981 + contentName);
1982 return;
1983 }
1984 final JinglePacket jinglePacket =
1985 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1986 send(jinglePacket);
1987 }
1988
1989 public RtpEndUserState getEndUserState() {
1990 switch (this.state) {
1991 case NULL, PROPOSED, SESSION_INITIALIZED -> {
1992 if (isInitiator()) {
1993 return RtpEndUserState.RINGING;
1994 } else {
1995 return RtpEndUserState.INCOMING_CALL;
1996 }
1997 }
1998 case PROCEED -> {
1999 if (isInitiator()) {
2000 return RtpEndUserState.RINGING;
2001 } else {
2002 return RtpEndUserState.ACCEPTING_CALL;
2003 }
2004 }
2005 case SESSION_INITIALIZED_PRE_APPROVED -> {
2006 if (isInitiator()) {
2007 return RtpEndUserState.RINGING;
2008 } else {
2009 return RtpEndUserState.CONNECTING;
2010 }
2011 }
2012 case SESSION_ACCEPTED -> {
2013 final ContentAddition ca = getPendingContentAddition();
2014 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2015 return RtpEndUserState.INCOMING_CONTENT_ADD;
2016 }
2017 return getPeerConnectionStateAsEndUserState();
2018 }
2019 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2020 if (isInitiator()) {
2021 return RtpEndUserState.DECLINED_OR_BUSY;
2022 } else {
2023 return RtpEndUserState.ENDED;
2024 }
2025 }
2026 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2027 return RtpEndUserState.ENDED;
2028 }
2029 case RETRACTED_RACED -> {
2030 if (isInitiator()) {
2031 return RtpEndUserState.ENDED;
2032 } else {
2033 return RtpEndUserState.RETRACTED;
2034 }
2035 }
2036 case TERMINATED_CONNECTIVITY_ERROR -> {
2037 return zeroDuration()
2038 ? RtpEndUserState.CONNECTIVITY_ERROR
2039 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2040 }
2041 case TERMINATED_APPLICATION_FAILURE -> {
2042 return RtpEndUserState.APPLICATION_ERROR;
2043 }
2044 case TERMINATED_SECURITY_ERROR -> {
2045 return RtpEndUserState.SECURITY_ERROR;
2046 }
2047 }
2048 throw new IllegalStateException(
2049 String.format("%s has no equivalent EndUserState", this.state));
2050 }
2051
2052 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2053 final PeerConnection.PeerConnectionState state;
2054 try {
2055 state = webRTCWrapper.getState();
2056 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2057 // We usually close the WebRTCWrapper *before* transitioning so we might still
2058 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2059 return RtpEndUserState.ENDING_CALL;
2060 }
2061 return switch (state) {
2062 case CONNECTED -> RtpEndUserState.CONNECTED;
2063 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2064 case CLOSED -> RtpEndUserState.ENDING_CALL;
2065 default -> zeroDuration()
2066 ? RtpEndUserState.CONNECTIVITY_ERROR
2067 : RtpEndUserState.RECONNECTING;
2068 };
2069 }
2070
2071 private boolean isPeerConnectionConnected() {
2072 try {
2073 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2074 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2075 return false;
2076 }
2077 }
2078
2079 private void updateCallIntegrationState() {
2080 switch (this.state) {
2081 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2082 if (isInitiator()) {
2083 this.callIntegration.setDialing();
2084 } else {
2085 this.callIntegration.setRinging();
2086 }
2087 }
2088 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2089 if (isInitiator()) {
2090 this.callIntegration.setDialing();
2091 } else {
2092 this.callIntegration.setInitialized();
2093 }
2094 }
2095 case SESSION_ACCEPTED -> {
2096 if (isPeerConnectionConnected()) {
2097 this.callIntegration.setActive();
2098 } else {
2099 this.callIntegration.setInitialized();
2100 }
2101 }
2102 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2103 if (isInitiator()) {
2104 this.callIntegration.busy();
2105 } else {
2106 this.callIntegration.rejected();
2107 }
2108 }
2109 case TERMINATED_SUCCESS -> this.callIntegration.success();
2110 case ACCEPTED -> this.callIntegration.accepted();
2111 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2112 .retracted();
2113 case TERMINATED_CONNECTIVITY_ERROR,
2114 TERMINATED_APPLICATION_FAILURE,
2115 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2116 default -> throw new IllegalStateException(
2117 String.format("%s is not handled", this.state));
2118 }
2119 }
2120
2121 public ContentAddition getPendingContentAddition() {
2122 final RtpContentMap in = this.incomingContentAdd;
2123 final RtpContentMap out = this.outgoingContentAdd;
2124 if (out != null) {
2125 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2126 } else if (in != null) {
2127 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2128 } else {
2129 return null;
2130 }
2131 }
2132
2133 public Set<Media> getMedia() {
2134 final State current = getState();
2135 if (current == State.NULL) {
2136 if (isInitiator()) {
2137 return Preconditions.checkNotNull(
2138 this.proposedMedia, "RTP connection has not been initialized properly");
2139 }
2140 throw new IllegalStateException("RTP connection has not been initialized yet");
2141 }
2142 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2143 return Preconditions.checkNotNull(
2144 this.proposedMedia, "RTP connection has not been initialized properly");
2145 }
2146 final RtpContentMap localContentMap = getLocalContentMap();
2147 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2148 if (localContentMap != null) {
2149 return localContentMap.getMedia();
2150 } else if (initiatorContentMap != null) {
2151 return initiatorContentMap.getMedia();
2152 } else if (isTerminated()) {
2153 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2154 } else {
2155 return Preconditions.checkNotNull(
2156 this.proposedMedia, "RTP connection has not been initialized properly");
2157 }
2158 }
2159
2160 public boolean isVerified() {
2161 final String fingerprint = this.omemoVerification.getFingerprint();
2162 if (fingerprint == null) {
2163 return false;
2164 }
2165 final FingerprintStatus status =
2166 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2167 return status != null && status.isVerified();
2168 }
2169
2170 public boolean addMedia(final Media media) {
2171 final Set<Media> currentMedia = getMedia();
2172 if (currentMedia.contains(media)) {
2173 throw new IllegalStateException(String.format("%s has already been proposed", media));
2174 }
2175 // TODO add state protection - can only add while ACCEPTED or so
2176 Log.d(Config.LOGTAG, "adding media: " + media);
2177 return webRTCWrapper.addTrack(media);
2178 }
2179
2180 public synchronized void acceptCall() {
2181 switch (this.state) {
2182 case PROPOSED -> {
2183 cancelRingingTimeout();
2184 acceptCallFromProposed();
2185 }
2186 case SESSION_INITIALIZED -> {
2187 cancelRingingTimeout();
2188 acceptCallFromSessionInitialized();
2189 }
2190 case ACCEPTED -> Log.w(
2191 Config.LOGTAG,
2192 id.account.getJid().asBareJid()
2193 + ": the call has already been accepted with another client. UI was just lagging behind");
2194 case PROCEED, SESSION_ACCEPTED -> Log.w(
2195 Config.LOGTAG,
2196 id.account.getJid().asBareJid()
2197 + ": the call has already been accepted. user probably double tapped the UI");
2198 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2199 }
2200 }
2201
2202 public synchronized void rejectCall() {
2203 if (isTerminated()) {
2204 Log.w(
2205 Config.LOGTAG,
2206 id.account.getJid().asBareJid()
2207 + ": received rejectCall() when session has already been terminated. nothing to do");
2208 return;
2209 }
2210 switch (this.state) {
2211 case PROPOSED -> rejectCallFromProposed();
2212 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2213 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2214 }
2215 }
2216
2217 public synchronized void endCall() {
2218 if (isTerminated()) {
2219 Log.w(
2220 Config.LOGTAG,
2221 id.account.getJid().asBareJid()
2222 + ": received endCall() when session has already been terminated. nothing to do");
2223 return;
2224 }
2225 if (isInState(State.PROPOSED) && isResponder()) {
2226 rejectCallFromProposed();
2227 return;
2228 }
2229 if (isInState(State.PROCEED)) {
2230 if (isInitiator()) {
2231 retractFromProceed();
2232 } else {
2233 rejectCallFromProceed();
2234 }
2235 return;
2236 }
2237 if (isInitiator()
2238 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2239 this.webRTCWrapper.close();
2240 sendSessionTerminate(Reason.CANCEL);
2241 return;
2242 }
2243 if (isInState(State.SESSION_INITIALIZED)) {
2244 rejectCallFromSessionInitiate();
2245 return;
2246 }
2247 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2248 this.webRTCWrapper.close();
2249 sendSessionTerminate(Reason.SUCCESS);
2250 return;
2251 }
2252 if (isInState(
2253 State.TERMINATED_APPLICATION_FAILURE,
2254 State.TERMINATED_CONNECTIVITY_ERROR,
2255 State.TERMINATED_DECLINED_OR_BUSY)) {
2256 Log.d(
2257 Config.LOGTAG,
2258 "ignoring request to end call because already in state " + this.state);
2259 return;
2260 }
2261 throw new IllegalStateException(
2262 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2263 }
2264
2265 private void retractFromProceed() {
2266 Log.d(Config.LOGTAG, "retract from proceed");
2267 this.sendJingleMessage("retract");
2268 closeTransitionLogFinish(State.RETRACTED_RACED);
2269 }
2270
2271 private void closeTransitionLogFinish(final State state) {
2272 this.webRTCWrapper.close();
2273 transitionOrThrow(state);
2274 writeLogMessage(state);
2275 finish();
2276 }
2277
2278 private void setupWebRTC(
2279 final Set<Media> media,
2280 final List<PeerConnection.IceServer> iceServers,
2281 final boolean trickle)
2282 throws WebRTCWrapper.InitializationException {
2283 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2284 this.webRTCWrapper.setup(
2285 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2286 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2287 }
2288
2289 private void acceptCallFromProposed() {
2290 transitionOrThrow(State.PROCEED);
2291 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2292 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2293 this.sendJingleMessage("proceed");
2294 }
2295
2296 private void rejectCallFromProposed() {
2297 transitionOrThrow(State.REJECTED);
2298 writeLogMessageMissed();
2299 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2300 this.sendJingleMessage("reject");
2301 finish();
2302 }
2303
2304 private void rejectCallFromProceed() {
2305 this.sendJingleMessage("reject");
2306 closeTransitionLogFinish(State.REJECTED_RACED);
2307 }
2308
2309 private void rejectCallFromSessionInitiate() {
2310 webRTCWrapper.close();
2311 sendSessionTerminate(Reason.DECLINE);
2312 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2313 }
2314
2315 private void sendJingleMessage(final String action) {
2316 sendJingleMessage(action, id.with);
2317 }
2318
2319 private void sendJingleMessage(final String action, final Jid to) {
2320 final MessagePacket messagePacket = new MessagePacket();
2321 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2322 messagePacket.setTo(to);
2323 final Element intent =
2324 messagePacket
2325 .addChild(action, Namespace.JINGLE_MESSAGE)
2326 .setAttribute("id", id.sessionId);
2327 if ("proceed".equals(action)) {
2328 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2329 if (isOmemoEnabled()) {
2330 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2331 final Element device =
2332 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2333 device.setAttribute("id", deviceId);
2334 }
2335 }
2336 messagePacket.addChild("store", "urn:xmpp:hints");
2337 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2338 }
2339
2340 private boolean isOmemoEnabled() {
2341 final Conversational conversational = message.getConversation();
2342 if (conversational instanceof Conversation) {
2343 return ((Conversation) conversational).getNextEncryption()
2344 == Message.ENCRYPTION_AXOLOTL;
2345 }
2346 return false;
2347 }
2348
2349 private void acceptCallFromSessionInitialized() {
2350 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2351 sendSessionAccept();
2352 }
2353
2354
2355 @Override
2356 protected synchronized boolean transition(final State target, final Runnable runnable) {
2357 if (super.transition(target, runnable)) {
2358 updateEndUserState();
2359 updateOngoingCallNotification();
2360 return true;
2361 } else {
2362 return false;
2363 }
2364 }
2365
2366 @Override
2367 public void onIceCandidate(final IceCandidate iceCandidate) {
2368 final RtpContentMap rtpContentMap =
2369 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2370 final IceUdpTransportInfo.Credentials credentials;
2371 try {
2372 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2373 } catch (final IllegalArgumentException e) {
2374 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2375 return;
2376 }
2377 final String uFrag = credentials.ufrag;
2378 final IceUdpTransportInfo.Candidate candidate =
2379 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2380 if (candidate == null) {
2381 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2382 return;
2383 }
2384 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2385 sendTransportInfo(iceCandidate.sdpMid, candidate);
2386 }
2387
2388 @Override
2389 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2390 Log.d(
2391 Config.LOGTAG,
2392 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2393 this.stateHistory.add(newState);
2394 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2395 this.sessionDuration.start();
2396 updateOngoingCallNotification();
2397 } else if (this.sessionDuration.isRunning()) {
2398 this.sessionDuration.stop();
2399 updateOngoingCallNotification();
2400 }
2401
2402 final boolean neverConnected =
2403 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2404
2405 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2406 if (neverConnected) {
2407 if (isTerminated()) {
2408 Log.d(
2409 Config.LOGTAG,
2410 id.account.getJid().asBareJid()
2411 + ": not sending session-terminate after connectivity error because session is already in state "
2412 + this.state);
2413 return;
2414 }
2415 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2416 return;
2417 } else {
2418 this.restartIce();
2419 }
2420 }
2421 updateEndUserState();
2422 }
2423
2424 private void restartIce() {
2425 this.stateHistory.clear();
2426 this.webRTCWrapper.restartIceAsync();
2427 }
2428
2429 @Override
2430 public void onRenegotiationNeeded() {
2431 this.webRTCWrapper.execute(this::renegotiate);
2432 }
2433
2434 private void renegotiate() {
2435 final SessionDescription sessionDescription;
2436 try {
2437 sessionDescription = setLocalSessionDescription();
2438 } catch (final Exception e) {
2439 final Throwable cause = Throwables.getRootCause(e);
2440 webRTCWrapper.close();
2441 if (isTerminated()) {
2442 Log.d(
2443 Config.LOGTAG,
2444 "failed to renegotiate. session was already terminated",
2445 cause);
2446 return;
2447 }
2448 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2449 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2450 return;
2451 }
2452 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2453 final RtpContentMap currentContentMap = getLocalContentMap();
2454 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2455 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2456
2457 Log.d(
2458 Config.LOGTAG,
2459 id.getAccount().getJid().asBareJid()
2460 + ": renegotiate. iceRestart="
2461 + iceRestart
2462 + " content id diff="
2463 + diff);
2464
2465 if (diff.hasModifications() && iceRestart) {
2466 webRTCWrapper.close();
2467 sendSessionTerminate(
2468 Reason.FAILED_APPLICATION,
2469 "WebRTC unexpectedly tried to modify content and transport at once");
2470 return;
2471 }
2472
2473 if (iceRestart) {
2474 initiateIceRestart(rtpContentMap);
2475 return;
2476 } else if (diff.isEmpty()) {
2477 Log.d(
2478 Config.LOGTAG,
2479 "renegotiation. nothing to do. SignalingState="
2480 + this.webRTCWrapper.getSignalingState());
2481 }
2482
2483 if (diff.added.size() > 0) {
2484 modifyLocalContentMap(rtpContentMap);
2485 sendContentAdd(rtpContentMap, diff.added);
2486 }
2487 }
2488
2489 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2490 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2491 final JinglePacket jinglePacket =
2492 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2493 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2494 jinglePacket.setTo(id.with);
2495 xmppConnectionService.sendIqPacket(
2496 id.account,
2497 jinglePacket,
2498 (account, response) -> {
2499 if (response.getType() == IqPacket.TYPE.RESULT) {
2500 Log.d(Config.LOGTAG, "received success to our ice restart");
2501 setLocalContentMap(rtpContentMap);
2502 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2503 return;
2504 }
2505 if (response.getType() == IqPacket.TYPE.ERROR) {
2506 if (isTieBreak(response)) {
2507 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2508 return;
2509 }
2510 handleIqErrorResponse(response);
2511 }
2512 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2513 handleIqTimeoutResponse(response);
2514 }
2515 });
2516 }
2517
2518 private boolean isTieBreak(final IqPacket response) {
2519 final Element error = response.findChild("error");
2520 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2521 }
2522
2523 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2524 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2525 this.outgoingContentAdd = contentAdd;
2526 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2527 prepareOutgoingContentMap(contentAdd);
2528 Futures.addCallback(
2529 outgoingContentMapFuture,
2530 new FutureCallback<>() {
2531 @Override
2532 public void onSuccess(final RtpContentMap outgoingContentMap) {
2533 sendContentAdd(outgoingContentMap);
2534 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2535 }
2536
2537 @Override
2538 public void onFailure(@NonNull Throwable throwable) {
2539 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2540 }
2541 },
2542 MoreExecutors.directExecutor());
2543 }
2544
2545 private void sendContentAdd(final RtpContentMap contentAdd) {
2546
2547 final JinglePacket jinglePacket =
2548 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2549 jinglePacket.setTo(id.with);
2550 xmppConnectionService.sendIqPacket(
2551 id.account,
2552 jinglePacket,
2553 (connection, response) -> {
2554 if (response.getType() == IqPacket.TYPE.RESULT) {
2555 Log.d(
2556 Config.LOGTAG,
2557 id.getAccount().getJid().asBareJid()
2558 + ": received ACK to our content-add");
2559 return;
2560 }
2561 if (response.getType() == IqPacket.TYPE.ERROR) {
2562 if (isTieBreak(response)) {
2563 this.outgoingContentAdd = null;
2564 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2565 return;
2566 }
2567 handleIqErrorResponse(response);
2568 }
2569 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2570 handleIqTimeoutResponse(response);
2571 }
2572 });
2573 }
2574
2575 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2576 if (isInitiator()) {
2577 this.initiatorRtpContentMap = rtpContentMap;
2578 } else {
2579 this.responderRtpContentMap = rtpContentMap;
2580 }
2581 }
2582
2583 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2584 if (isInitiator()) {
2585 this.responderRtpContentMap = rtpContentMap;
2586 } else {
2587 this.initiatorRtpContentMap = rtpContentMap;
2588 }
2589 }
2590
2591 // this method is to be used for content map modifications that modify media
2592 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2593 final RtpContentMap activeContents = rtpContentMap.activeContents();
2594 setLocalContentMap(activeContents);
2595 // TODO change audio device on callIntegration was (`switchSpeakerPhonePreference(AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia())`)
2596 updateEndUserState();
2597 }
2598
2599 private SessionDescription setLocalSessionDescription()
2600 throws ExecutionException, InterruptedException {
2601 final org.webrtc.SessionDescription sessionDescription =
2602 this.webRTCWrapper.setLocalDescription(false).get();
2603 return SessionDescription.parse(sessionDescription.description);
2604 }
2605
2606 private void closeWebRTCSessionAfterFailedConnection() {
2607 this.webRTCWrapper.close();
2608 synchronized (this) {
2609 if (isTerminated()) {
2610 Log.d(
2611 Config.LOGTAG,
2612 id.account.getJid().asBareJid()
2613 + ": no need to send session-terminate after failed connection. Other party already did");
2614 return;
2615 }
2616 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2617 }
2618 }
2619
2620 public boolean zeroDuration() {
2621 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2622 }
2623
2624 public long getCallDuration() {
2625 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2626 }
2627
2628
2629 public CallIntegration getCallIntegration() {
2630 return this.callIntegration;
2631 }
2632
2633 public boolean isMicrophoneEnabled() {
2634 return webRTCWrapper.isMicrophoneEnabled();
2635 }
2636
2637 public boolean setMicrophoneEnabled(final boolean enabled) {
2638 return webRTCWrapper.setMicrophoneEnabled(enabled);
2639 }
2640
2641 public boolean isVideoEnabled() {
2642 return webRTCWrapper.isVideoEnabled();
2643 }
2644
2645 public void setVideoEnabled(final boolean enabled) {
2646 webRTCWrapper.setVideoEnabled(enabled);
2647 }
2648
2649 public boolean isCameraSwitchable() {
2650 return webRTCWrapper.isCameraSwitchable();
2651 }
2652
2653 public boolean isFrontCamera() {
2654 return webRTCWrapper.isFrontCamera();
2655 }
2656
2657 public ListenableFuture<Boolean> switchCamera() {
2658 return webRTCWrapper.switchCamera();
2659 }
2660
2661 @Override
2662 public void onCallIntegrationShowIncomingCallUi() {
2663 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2664 }
2665
2666 @Override
2667 public void onCallIntegrationDisconnect() {
2668 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2669 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2670 rejectCall();
2671 } else {
2672 endCall();
2673 }
2674 }
2675
2676 @Override
2677 public void onAudioDeviceChanged(
2678 final CallIntegration.AudioDevice selectedAudioDevice,
2679 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2680 Log.d(Config.LOGTAG,"onAudioDeviceChanged("+selectedAudioDevice+","+availableAudioDevices+")");
2681 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2682 selectedAudioDevice, availableAudioDevices);
2683 }
2684
2685 private void updateEndUserState() {
2686 final RtpEndUserState endUserState = getEndUserState();
2687 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2688 this.updateCallIntegrationState();
2689 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2690 id.account, id.with, id.sessionId, endUserState);
2691 }
2692
2693 private void updateOngoingCallNotification() {
2694 final State state = this.state;
2695 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2696 final boolean reconnecting;
2697 if (state == State.SESSION_ACCEPTED) {
2698 reconnecting =
2699 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2700 } else {
2701 reconnecting = false;
2702 }
2703 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2704 } else {
2705 xmppConnectionService.removeOngoingCall();
2706 }
2707 }
2708
2709 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2710 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2711 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2712 request.setTo(id.account.getDomain());
2713 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2714 xmppConnectionService.sendIqPacket(
2715 id.account,
2716 request,
2717 (account, response) -> {
2718 final var iceServers = IceServers.parse(response);
2719 if (iceServers.size() == 0) {
2720 Log.w(
2721 Config.LOGTAG,
2722 id.account.getJid().asBareJid()
2723 + ": no ICE server found "
2724 + response);
2725 }
2726 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2727 });
2728 } else {
2729 Log.w(
2730 Config.LOGTAG,
2731 id.account.getJid().asBareJid() + ": has no external service discovery");
2732 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2733 }
2734 }
2735
2736 @Override
2737 protected void terminateTransport() {
2738 this.webRTCWrapper.close();
2739 }
2740
2741 @Override
2742 protected void finish() {
2743 if (isTerminated()) {
2744 this.cancelRingingTimeout();
2745 this.callIntegration.verifyDisconnected();
2746 this.webRTCWrapper.verifyClosed();
2747 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2748 super.finish();
2749 } else {
2750 throw new IllegalStateException(
2751 String.format("Unable to call finish from %s", this.state));
2752 }
2753 }
2754
2755 private void writeLogMessage(final State state) {
2756 final long duration = getCallDuration();
2757 if (state == State.TERMINATED_SUCCESS
2758 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2759 writeLogMessageSuccess(duration);
2760 } else {
2761 writeLogMessageMissed();
2762 }
2763 }
2764
2765 private void writeLogMessageSuccess(final long duration) {
2766 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2767 this.writeMessage();
2768 }
2769
2770 private void writeLogMessageMissed() {
2771 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2772 this.writeMessage();
2773 }
2774
2775 private void writeMessage() {
2776 final Conversational conversational = message.getConversation();
2777 if (conversational instanceof Conversation) {
2778 ((Conversation) conversational).add(this.message);
2779 xmppConnectionService.createMessageAsync(message);
2780 xmppConnectionService.updateConversationUi();
2781 } else {
2782 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2783 }
2784 }
2785
2786 public Optional<VideoTrack> getLocalVideoTrack() {
2787 return webRTCWrapper.getLocalVideoTrack();
2788 }
2789
2790 public Optional<VideoTrack> getRemoteVideoTrack() {
2791 return webRTCWrapper.getRemoteVideoTrack();
2792 }
2793
2794 public EglBase.Context getEglBaseContext() {
2795 return webRTCWrapper.getEglBaseContext();
2796 }
2797
2798 void setProposedMedia(final Set<Media> media) {
2799 this.proposedMedia = media;
2800 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2801 }
2802
2803 public void fireStateUpdate() {
2804 final RtpEndUserState endUserState = getEndUserState();
2805 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2806 id.account, id.with, id.sessionId, endUserState);
2807 }
2808
2809 public boolean isSwitchToVideoAvailable() {
2810 final boolean prerequisite =
2811 Media.audioOnly(getMedia())
2812 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2813 .contains(getEndUserState());
2814 return prerequisite && remoteHasVideoFeature();
2815 }
2816
2817 private boolean remoteHasVideoFeature() {
2818 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2819 }
2820
2821 private boolean remoteHasSdpOfferAnswer() {
2822 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2823 }
2824
2825 private interface OnIceServersDiscovered {
2826 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2827 }
2828}