1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 public static final List<State> STATES_SHOWING_ONGOING_CALL =
80 Arrays.asList(
81 State.PROPOSED,
82 State.PROCEED,
83 State.SESSION_INITIALIZED_PRE_APPROVED,
84 State.SESSION_ACCEPTED);
85 private static final long BUSY_TIME_OUT = 30;
86
87 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
88 private final Queue<
89 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
90 pendingIceCandidates = new LinkedList<>();
91 private final OmemoVerification omemoVerification = new OmemoVerification();
92 public final CallIntegration callIntegration;
93 private final Message message;
94
95 private Set<Media> proposedMedia;
96 private RtpContentMap initiatorRtpContentMap;
97 private RtpContentMap responderRtpContentMap;
98 private RtpContentMap incomingContentAdd;
99 private RtpContentMap outgoingContentAdd;
100 private IceUdpTransportInfo.Setup peerDtlsSetup;
101 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
102 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
103 private ScheduledFuture<?> ringingTimeoutFuture;
104 private final long created = System.currentTimeMillis() / 1000L;
105
106 JingleRtpConnection(
107 final JingleConnectionManager jingleConnectionManager,
108 final Id id,
109 final Jid initiator) {
110 this(
111 jingleConnectionManager,
112 id,
113 initiator,
114 new CallIntegration(
115 jingleConnectionManager
116 .getXmppConnectionService()
117 .getApplicationContext()));
118 this.callIntegration.setAddress(
119 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
120 this.callIntegration.setInitialized();
121 }
122
123 JingleRtpConnection(
124 final JingleConnectionManager jingleConnectionManager,
125 final Id id,
126 final Jid initiator,
127 final CallIntegration callIntegration) {
128 super(jingleConnectionManager, id, initiator);
129 final Conversation conversation =
130 jingleConnectionManager
131 .getXmppConnectionService()
132 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
133 this.message =
134 new Message(
135 conversation,
136 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
137 Message.TYPE_RTP_SESSION,
138 id.sessionId);
139 this.callIntegration = callIntegration;
140 this.callIntegration.setCallback(this);
141 }
142
143 @Override
144 synchronized void deliverPacket(final JinglePacket jinglePacket) {
145 switch (jinglePacket.getAction()) {
146 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
147 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
148 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
149 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
150 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
151 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
152 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
153 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
154 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
155 default -> {
156 respondOk(jinglePacket);
157 Log.d(
158 Config.LOGTAG,
159 String.format(
160 "%s: received unhandled jingle action %s",
161 id.account.getJid().asBareJid(), jinglePacket.getAction()));
162 }
163 }
164 }
165
166 @Override
167 synchronized void notifyRebound() {
168 if (isTerminated()) {
169 return;
170 }
171 webRTCWrapper.close();
172 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
173 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
174 }
175 if (isInState(
176 State.SESSION_INITIALIZED,
177 State.SESSION_INITIALIZED_PRE_APPROVED,
178 State.SESSION_ACCEPTED)) {
179 // we might have already changed resources (full jid) at this point; so this might not
180 // even reach the other party
181 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
182 } else {
183 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
184 finish();
185 }
186 }
187
188 public boolean applyDtmfTone(final String tone) {
189 return webRTCWrapper.applyDtmfTone(tone);
190 }
191
192 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
193 respondOk(jinglePacket);
194 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
195 final State previous = this.state;
196 Log.d(
197 Config.LOGTAG,
198 id.account.getJid().asBareJid()
199 + ": received session terminate reason="
200 + wrapper.reason
201 + "("
202 + Strings.nullToEmpty(wrapper.text)
203 + ") while in state "
204 + previous);
205 if (TERMINATED.contains(previous)) {
206 Log.d(
207 Config.LOGTAG,
208 id.account.getJid().asBareJid()
209 + ": ignoring session terminate because already in "
210 + previous);
211 return;
212 }
213 webRTCWrapper.close();
214 final State target = reasonToState(wrapper.reason);
215 transitionOrThrow(target);
216 writeLogMessage(target);
217 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
218 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
219 }
220 finish();
221 }
222
223 private void receiveTransportInfo(final JinglePacket jinglePacket) {
224 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
225 // INITIALIZED only after transport-info has been received
226 if (isInState(
227 State.NULL,
228 State.PROCEED,
229 State.SESSION_INITIALIZED,
230 State.SESSION_INITIALIZED_PRE_APPROVED,
231 State.SESSION_ACCEPTED)) {
232 final RtpContentMap contentMap;
233 try {
234 contentMap = RtpContentMap.of(jinglePacket);
235 } catch (final IllegalArgumentException | NullPointerException e) {
236 Log.d(
237 Config.LOGTAG,
238 id.account.getJid().asBareJid()
239 + ": improperly formatted contents; ignoring",
240 e);
241 respondOk(jinglePacket);
242 return;
243 }
244 receiveTransportInfo(jinglePacket, contentMap);
245 } else {
246 if (isTerminated()) {
247 respondOk(jinglePacket);
248 Log.d(
249 Config.LOGTAG,
250 id.account.getJid().asBareJid()
251 + ": ignoring out-of-order transport info; we where already terminated");
252 } else {
253 Log.d(
254 Config.LOGTAG,
255 id.account.getJid().asBareJid()
256 + ": received transport info while in state="
257 + this.state);
258 terminateWithOutOfOrder(jinglePacket);
259 }
260 }
261 }
262
263 private void receiveTransportInfo(
264 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
265 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
266 candidates = contentMap.contents.entrySet();
267 final RtpContentMap remote = getRemoteContentMap();
268 final Set<String> remoteContentIds =
269 remote == null ? Collections.emptySet() : remote.contents.keySet();
270 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
271 Log.d(
272 Config.LOGTAG,
273 "received transport-info for unknown contents "
274 + contentMap.contents.keySet()
275 + " (known: "
276 + remoteContentIds
277 + ")");
278 respondOk(jinglePacket);
279 pendingIceCandidates.addAll(candidates);
280 return;
281 }
282 if (this.state != State.SESSION_ACCEPTED) {
283 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
284 respondOk(jinglePacket);
285 pendingIceCandidates.addAll(candidates);
286 return;
287 }
288 // zero candidates + modified credentials are an ICE restart offer
289 if (checkForIceRestart(jinglePacket, contentMap)) {
290 return;
291 }
292 respondOk(jinglePacket);
293 try {
294 processCandidates(candidates);
295 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
296 Log.w(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
300 }
301 }
302
303 private void receiveContentAdd(final JinglePacket jinglePacket) {
304 final RtpContentMap modification;
305 try {
306 modification = RtpContentMap.of(jinglePacket);
307 modification.requireContentDescriptions();
308 } catch (final RuntimeException e) {
309 Log.d(
310 Config.LOGTAG,
311 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
312 Throwables.getRootCause(e));
313 respondOk(jinglePacket);
314 webRTCWrapper.close();
315 sendSessionTerminate(Reason.of(e), e.getMessage());
316 return;
317 }
318 if (isInState(State.SESSION_ACCEPTED)) {
319 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
320 final ListenableFuture<RtpContentMap> future =
321 receiveRtpContentMap(
322 modification,
323 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
324 Futures.addCallback(
325 future,
326 new FutureCallback<>() {
327 @Override
328 public void onSuccess(final RtpContentMap rtpContentMap) {
329 receiveContentAdd(jinglePacket, rtpContentMap);
330 }
331
332 @Override
333 public void onFailure(@NonNull Throwable throwable) {
334 respondOk(jinglePacket);
335 final Throwable rootCause = Throwables.getRootCause(throwable);
336 Log.d(
337 Config.LOGTAG,
338 id.account.getJid().asBareJid()
339 + ": improperly formatted contents in content-add",
340 throwable);
341 webRTCWrapper.close();
342 sendSessionTerminate(
343 Reason.ofThrowable(rootCause), rootCause.getMessage());
344 }
345 },
346 MoreExecutors.directExecutor());
347 } else {
348 terminateWithOutOfOrder(jinglePacket);
349 }
350 }
351
352 private void receiveContentAdd(
353 final JinglePacket jinglePacket, final RtpContentMap modification) {
354 final RtpContentMap remote = getRemoteContentMap();
355 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
356 respondOk(jinglePacket);
357 this.webRTCWrapper.close();
358 sendSessionTerminate(
359 Reason.FAILED_APPLICATION,
360 String.format(
361 "contents with names %s already exists",
362 Joiner.on(", ").join(modification.getNames())));
363 return;
364 }
365 final ContentAddition contentAddition =
366 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
367
368 final RtpContentMap outgoing = this.outgoingContentAdd;
369 final Set<ContentAddition.Summary> outgoingContentAddSummary =
370 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
371
372 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
373 if (isInitiator()) {
374 Log.d(
375 Config.LOGTAG,
376 id.getAccount().getJid().asBareJid()
377 + ": respond with tie break to matching content-add offer");
378 respondWithTieBreak(jinglePacket);
379 } else {
380 Log.d(
381 Config.LOGTAG,
382 id.getAccount().getJid().asBareJid()
383 + ": automatically accept matching content-add offer");
384 acceptContentAdd(contentAddition.summary, modification);
385 }
386 return;
387 }
388
389 // once we can display multiple video tracks we can be more loose with this condition
390 // theoretically it should also be fine to automatically accept audio only contents
391 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
392 Log.d(
393 Config.LOGTAG,
394 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
395 this.incomingContentAdd = modification;
396 respondOk(jinglePacket);
397 updateEndUserState();
398 } else {
399 respondOk(jinglePacket);
400 // TODO do we want to add a reason?
401 rejectContentAdd(modification);
402 }
403 }
404
405 private void receiveContentAccept(final JinglePacket jinglePacket) {
406 final RtpContentMap receivedContentAccept;
407 try {
408 receivedContentAccept = RtpContentMap.of(jinglePacket);
409 receivedContentAccept.requireContentDescriptions();
410 } catch (final RuntimeException e) {
411 Log.d(
412 Config.LOGTAG,
413 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
414 Throwables.getRootCause(e));
415 respondOk(jinglePacket);
416 webRTCWrapper.close();
417 sendSessionTerminate(Reason.of(e), e.getMessage());
418 return;
419 }
420
421 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
422 if (outgoingContentAdd == null) {
423 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
424 terminateWithOutOfOrder(jinglePacket);
425 return;
426 }
427 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
428 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
429 this.outgoingContentAdd = null;
430 respondOk(jinglePacket);
431 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
432 final ListenableFuture<RtpContentMap> future =
433 receiveRtpContentMap(
434 receivedContentAccept,
435 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
436 Futures.addCallback(
437 future,
438 new FutureCallback<>() {
439 @Override
440 public void onSuccess(final RtpContentMap result) {
441 receiveContentAccept(result);
442 }
443
444 @Override
445 public void onFailure(@NonNull final Throwable throwable) {
446 webRTCWrapper.close();
447 sendSessionTerminate(
448 Reason.ofThrowable(throwable), throwable.getMessage());
449 }
450 },
451 MoreExecutors.directExecutor());
452 } else {
453 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
454 terminateWithOutOfOrder(jinglePacket);
455 }
456 }
457
458 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
459 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
460 final RtpContentMap modifiedContentMap =
461 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
462
463 setRemoteContentMap(modifiedContentMap);
464
465 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
466
467 final org.webrtc.SessionDescription sdp =
468 new org.webrtc.SessionDescription(
469 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
470
471 try {
472 this.webRTCWrapper.setRemoteDescription(sdp).get();
473 } catch (final Exception e) {
474 final Throwable cause = Throwables.getRootCause(e);
475 Log.d(
476 Config.LOGTAG,
477 id.getAccount().getJid().asBareJid()
478 + ": unable to set remote description after receiving content-accept",
479 cause);
480 webRTCWrapper.close();
481 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
482 return;
483 }
484 Log.d(
485 Config.LOGTAG,
486 id.getAccount().getJid().asBareJid()
487 + ": remote has accepted content-add "
488 + ContentAddition.summary(receivedContentAccept));
489 processCandidates(receivedContentAccept.contents.entrySet());
490 updateEndUserState();
491 }
492
493 private void receiveContentModify(final JinglePacket jinglePacket) {
494 if (this.state != State.SESSION_ACCEPTED) {
495 terminateWithOutOfOrder(jinglePacket);
496 return;
497 }
498 final Map<String, Content.Senders> modification =
499 Maps.transformEntries(
500 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
501 final boolean isInitiator = isInitiator();
502 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
503 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
504 final Set<String> currentOutgoingMediaIds =
505 currentOutgoing == null
506 ? Collections.emptySet()
507 : currentOutgoing.contents.keySet();
508 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
509 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
510 respondOk(jinglePacket);
511 final RtpContentMap modifiedContentMap;
512 try {
513 modifiedContentMap =
514 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
515 } catch (final IllegalArgumentException e) {
516 webRTCWrapper.close();
517 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
518 return;
519 }
520 this.outgoingContentAdd = modifiedContentMap;
521 Log.d(
522 Config.LOGTAG,
523 id.account.getJid().asBareJid()
524 + ": processed content-modification for pending content-add");
525 } else if (remoteContentMap != null
526 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
527 respondOk(jinglePacket);
528 final RtpContentMap modifiedRemoteContentMap;
529 try {
530 modifiedRemoteContentMap =
531 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
532 } catch (final IllegalArgumentException e) {
533 webRTCWrapper.close();
534 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
535 return;
536 }
537 final SessionDescription offer;
538 try {
539 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
540 } catch (final IllegalArgumentException | NullPointerException e) {
541 Log.d(
542 Config.LOGTAG,
543 id.getAccount().getJid().asBareJid()
544 + ": unable convert offer from content-modify to SDP",
545 e);
546 webRTCWrapper.close();
547 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
548 return;
549 }
550 Log.d(
551 Config.LOGTAG,
552 id.account.getJid().asBareJid() + ": auto accepting content-modification");
553 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
554 } else {
555 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
556 respondWithItemNotFound(jinglePacket);
557 }
558 }
559
560 private void autoAcceptContentModify(
561 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
562 this.setRemoteContentMap(modifiedRemoteContentMap);
563 final org.webrtc.SessionDescription sdp =
564 new org.webrtc.SessionDescription(
565 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
566 try {
567 this.webRTCWrapper.setRemoteDescription(sdp).get();
568 // auto accept is only done when we already have tracks
569 final SessionDescription answer = setLocalSessionDescription();
570 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
571 modifyLocalContentMap(rtpContentMap);
572 // we do not need to send an answer but do we have to resend the candidates currently in
573 // SDP?
574 // resendCandidatesFromSdp(answer);
575 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
576 } catch (final Exception e) {
577 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
578 webRTCWrapper.close();
579 sendSessionTerminate(Reason.FAILED_APPLICATION);
580 }
581 }
582
583 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
584 final SessionDescription answer) {
585 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
586 new ImmutableMultimap.Builder<>();
587 for (final SessionDescription.Media media : answer.media) {
588 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
589 if (Strings.isNullOrEmpty(mid)) {
590 continue;
591 }
592 for (final String sdpCandidate : media.attributes.get("candidate")) {
593 final IceUdpTransportInfo.Candidate candidate =
594 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
595 if (candidate != null) {
596 candidateBuilder.put(mid, candidate);
597 }
598 }
599 }
600 return candidateBuilder.build();
601 }
602
603 private void receiveContentReject(final JinglePacket jinglePacket) {
604 final RtpContentMap receivedContentReject;
605 try {
606 receivedContentReject = RtpContentMap.of(jinglePacket);
607 } catch (final RuntimeException e) {
608 Log.d(
609 Config.LOGTAG,
610 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
611 Throwables.getRootCause(e));
612 respondOk(jinglePacket);
613 this.webRTCWrapper.close();
614 sendSessionTerminate(Reason.of(e), e.getMessage());
615 return;
616 }
617
618 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
619 if (outgoingContentAdd == null) {
620 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
621 terminateWithOutOfOrder(jinglePacket);
622 return;
623 }
624 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
625 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
626 this.outgoingContentAdd = null;
627 respondOk(jinglePacket);
628 Log.d(Config.LOGTAG, jinglePacket.toString());
629 receiveContentReject(ourSummary);
630 } else {
631 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
632 terminateWithOutOfOrder(jinglePacket);
633 }
634 }
635
636 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
637 try {
638 this.webRTCWrapper.removeTrack(Media.VIDEO);
639 final RtpContentMap localContentMap = customRollback();
640 modifyLocalContentMap(localContentMap);
641 } catch (final Exception e) {
642 final Throwable cause = Throwables.getRootCause(e);
643 Log.d(
644 Config.LOGTAG,
645 id.getAccount().getJid().asBareJid()
646 + ": unable to rollback local description after receiving content-reject",
647 cause);
648 webRTCWrapper.close();
649 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
650 return;
651 }
652 Log.d(
653 Config.LOGTAG,
654 id.getAccount().getJid().asBareJid()
655 + ": remote has rejected our content-add "
656 + summary);
657 }
658
659 private void receiveContentRemove(final JinglePacket jinglePacket) {
660 final RtpContentMap receivedContentRemove;
661 try {
662 receivedContentRemove = RtpContentMap.of(jinglePacket);
663 receivedContentRemove.requireContentDescriptions();
664 } catch (final RuntimeException e) {
665 Log.d(
666 Config.LOGTAG,
667 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
668 Throwables.getRootCause(e));
669 respondOk(jinglePacket);
670 this.webRTCWrapper.close();
671 sendSessionTerminate(Reason.of(e), e.getMessage());
672 return;
673 }
674 respondOk(jinglePacket);
675 receiveContentRemove(receivedContentRemove);
676 }
677
678 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
679 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
680 final Set<ContentAddition.Summary> contentAddSummary =
681 incomingContentAdd == null
682 ? Collections.emptySet()
683 : ContentAddition.summary(incomingContentAdd);
684 final Set<ContentAddition.Summary> removeSummary =
685 ContentAddition.summary(receivedContentRemove);
686 if (contentAddSummary.equals(removeSummary)) {
687 this.incomingContentAdd = null;
688 updateEndUserState();
689 } else {
690 webRTCWrapper.close();
691 sendSessionTerminate(
692 Reason.FAILED_APPLICATION,
693 String.format(
694 "%s only supports %s as a means to retract a not yet accepted %s",
695 BuildConfig.APP_NAME,
696 JinglePacket.Action.CONTENT_REMOVE,
697 JinglePacket.Action.CONTENT_ADD));
698 }
699 }
700
701 public synchronized void retractContentAdd() {
702 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
703 if (outgoingContentAdd == null) {
704 throw new IllegalStateException("Not outgoing content add");
705 }
706 try {
707 webRTCWrapper.removeTrack(Media.VIDEO);
708 final RtpContentMap localContentMap = customRollback();
709 modifyLocalContentMap(localContentMap);
710 } catch (final Exception e) {
711 final Throwable cause = Throwables.getRootCause(e);
712 Log.d(
713 Config.LOGTAG,
714 id.getAccount().getJid().asBareJid()
715 + ": unable to rollback local description after trying to retract content-add",
716 cause);
717 webRTCWrapper.close();
718 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
719 return;
720 }
721 this.outgoingContentAdd = null;
722 final JinglePacket retract =
723 outgoingContentAdd
724 .toStub()
725 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
726 this.send(retract);
727 Log.d(
728 Config.LOGTAG,
729 id.getAccount().getJid()
730 + ": retract content-add "
731 + ContentAddition.summary(outgoingContentAdd));
732 }
733
734 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
735 final SessionDescription sdp = setLocalSessionDescription();
736 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
737 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
738 this.webRTCWrapper
739 .setRemoteDescription(
740 new org.webrtc.SessionDescription(
741 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
742 .get();
743 return localRtpContentMap;
744 }
745
746 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
747 final RtpContentMap currentRemote = getRemoteContentMap();
748 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
749 if (diff.isEmpty()) {
750 throw new IllegalStateException(
751 "Unexpected rollback condition. No difference between local and remote");
752 }
753 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
754 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
755 final RtpContentMap nextRemote =
756 currentRemote.addContent(
757 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
758 return SessionDescription.of(nextRemote, isResponder());
759 }
760 throw new IllegalStateException(
761 "Unexpected rollback condition. Senders were not uniformly none");
762 }
763
764 public synchronized void acceptContentAdd(
765 @NonNull final Set<ContentAddition.Summary> contentAddition) {
766 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
767 if (incomingContentAdd == null) {
768 throw new IllegalStateException("No incoming content add");
769 }
770
771 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
772 this.incomingContentAdd = null;
773 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
774 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
775 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
776 Log.d(
777 Config.LOGTAG,
778 "content addition is receive only. we want to upgrade to 'both'");
779 final RtpContentMap modifiedSenders =
780 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
781 final JinglePacket proposedContentModification =
782 modifiedSenders
783 .toStub()
784 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
785 proposedContentModification.setTo(id.with);
786 xmppConnectionService.sendIqPacket(
787 id.account,
788 proposedContentModification,
789 (account, response) -> {
790 if (response.getType() == IqPacket.TYPE.RESULT) {
791 Log.d(
792 Config.LOGTAG,
793 id.account.getJid().asBareJid()
794 + ": remote has accepted our upgrade to senders=both");
795 acceptContentAdd(
796 ContentAddition.summary(modifiedSenders), modifiedSenders);
797 } else {
798 Log.d(
799 Config.LOGTAG,
800 id.account.getJid().asBareJid()
801 + ": remote has rejected our upgrade to senders=both");
802 acceptContentAdd(contentAddition, incomingContentAdd);
803 }
804 });
805 } else {
806 acceptContentAdd(contentAddition, incomingContentAdd);
807 }
808 } else {
809 throw new IllegalStateException(
810 "Accepted content add does not match pending content-add");
811 }
812 }
813
814 private void acceptContentAdd(
815 @NonNull final Set<ContentAddition.Summary> contentAddition,
816 final RtpContentMap incomingContentAdd) {
817 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
818 final RtpContentMap modifiedContentMap =
819 getRemoteContentMap().addContent(incomingContentAdd, setup);
820 this.setRemoteContentMap(modifiedContentMap);
821
822 final SessionDescription offer;
823 try {
824 offer = SessionDescription.of(modifiedContentMap, isResponder());
825 } catch (final IllegalArgumentException | NullPointerException e) {
826 Log.d(
827 Config.LOGTAG,
828 id.getAccount().getJid().asBareJid()
829 + ": unable convert offer from content-add to SDP",
830 e);
831 webRTCWrapper.close();
832 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
833 return;
834 }
835 this.incomingContentAdd = null;
836 acceptContentAdd(contentAddition, offer);
837 }
838
839 private void acceptContentAdd(
840 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
841 final org.webrtc.SessionDescription sdp =
842 new org.webrtc.SessionDescription(
843 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
844 try {
845 this.webRTCWrapper.setRemoteDescription(sdp).get();
846
847 // TODO add tracks for 'media' where contentAddition.senders matches
848
849 // TODO if senders.sending(isInitiator())
850
851 this.webRTCWrapper.addTrack(Media.VIDEO);
852
853 // TODO add additional transceivers for recv only cases
854
855 final SessionDescription answer = setLocalSessionDescription();
856 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
857
858 final RtpContentMap contentAcceptMap =
859 rtpContentMap.toContentModification(
860 Collections2.transform(contentAddition, ca -> ca.name));
861
862 Log.d(
863 Config.LOGTAG,
864 id.getAccount().getJid().asBareJid()
865 + ": sending content-accept "
866 + ContentAddition.summary(contentAcceptMap));
867
868 addIceCandidatesFromBlackLog();
869
870 modifyLocalContentMap(rtpContentMap);
871 final ListenableFuture<RtpContentMap> future =
872 prepareOutgoingContentMap(contentAcceptMap);
873 Futures.addCallback(
874 future,
875 new FutureCallback<>() {
876 @Override
877 public void onSuccess(final RtpContentMap rtpContentMap) {
878 sendContentAccept(rtpContentMap);
879 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
880 }
881
882 @Override
883 public void onFailure(@NonNull final Throwable throwable) {
884 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
885 }
886 },
887 MoreExecutors.directExecutor());
888 } catch (final Exception e) {
889 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
890 webRTCWrapper.close();
891 sendSessionTerminate(Reason.FAILED_APPLICATION);
892 }
893 }
894
895 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
896 final JinglePacket jinglePacket =
897 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
898 send(jinglePacket);
899 }
900
901 public synchronized void rejectContentAdd() {
902 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
903 if (incomingContentAdd == null) {
904 throw new IllegalStateException("No incoming content add");
905 }
906 this.incomingContentAdd = null;
907 updateEndUserState();
908 rejectContentAdd(incomingContentAdd);
909 }
910
911 private void rejectContentAdd(final RtpContentMap contentMap) {
912 final JinglePacket jinglePacket =
913 contentMap
914 .toStub()
915 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
916 Log.d(
917 Config.LOGTAG,
918 id.getAccount().getJid().asBareJid()
919 + ": rejecting content "
920 + ContentAddition.summary(contentMap));
921 send(jinglePacket);
922 }
923
924 private boolean checkForIceRestart(
925 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
926 final RtpContentMap existing = getRemoteContentMap();
927 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
928 final IceUdpTransportInfo.Credentials newCredentials;
929 try {
930 existingCredentials = existing.getCredentials();
931 newCredentials = rtpContentMap.getDistinctCredentials();
932 } catch (final IllegalStateException e) {
933 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
934 return false;
935 }
936 if (existingCredentials.contains(newCredentials)) {
937 return false;
938 }
939 // TODO an alternative approach is to check if we already got an iq result to our
940 // ICE-restart
941 // and if that's the case we are seeing an answer.
942 // This might be more spec compliant but also more error prone potentially
943 final boolean isSignalStateStable =
944 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
945 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
946 final boolean isOffer = rtpContentMap.emptyCandidates();
947 final RtpContentMap restartContentMap;
948 try {
949 if (isOffer) {
950 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
951 restartContentMap =
952 existing.modifiedCredentials(
953 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
954 } else {
955 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
956 Log.d(
957 Config.LOGTAG,
958 "received confirmation of ICE restart"
959 + newCredentials
960 + " peer_setup="
961 + setup);
962 // DTLS setup attribute needs to be rewritten to reflect current peer state
963 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
964 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
965 }
966 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
967 return isOffer;
968 } else {
969 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
970 respondWithTieBreak(jinglePacket);
971 return true;
972 }
973 } catch (final Exception exception) {
974 respondOk(jinglePacket);
975 final Throwable rootCause = Throwables.getRootCause(exception);
976 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
977 // If this happens a termination is already in progress
978 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
979 return true;
980 }
981 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
982 webRTCWrapper.close();
983 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
984 return true;
985 }
986 }
987
988 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
989 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
990 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
991 throw new IllegalStateException("Invalid peer setup");
992 }
993 return peerSetup;
994 }
995
996 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
997 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
998 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
999 }
1000 this.peerDtlsSetup = setup;
1001 }
1002
1003 private boolean applyIceRestart(
1004 final JinglePacket jinglePacket,
1005 final RtpContentMap restartContentMap,
1006 final boolean isOffer)
1007 throws ExecutionException, InterruptedException {
1008 final SessionDescription sessionDescription =
1009 SessionDescription.of(restartContentMap, isResponder());
1010 final org.webrtc.SessionDescription.Type type =
1011 isOffer
1012 ? org.webrtc.SessionDescription.Type.OFFER
1013 : org.webrtc.SessionDescription.Type.ANSWER;
1014 org.webrtc.SessionDescription sdp =
1015 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1016 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1017 if (isInitiator()) {
1018 // We ignore the offer and respond with tie-break. This will clause the responder
1019 // not to apply the content map
1020 return false;
1021 }
1022 }
1023 webRTCWrapper.setRemoteDescription(sdp).get();
1024 setRemoteContentMap(restartContentMap);
1025 if (isOffer) {
1026 final SessionDescription localSessionDescription = setLocalSessionDescription();
1027 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1028 // We need to respond OK before sending any candidates
1029 respondOk(jinglePacket);
1030 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1031 } else {
1032 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1033 }
1034 return true;
1035 }
1036
1037 private void processCandidates(
1038 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1039 contents) {
1040 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1041 content : contents) {
1042 processCandidate(content);
1043 }
1044 }
1045
1046 private void processCandidate(
1047 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1048 content) {
1049 final RtpContentMap rtpContentMap = getRemoteContentMap();
1050 final List<String> indices = toIdentificationTags(rtpContentMap);
1051 final String sdpMid = content.getKey(); // aka content name
1052 final IceUdpTransportInfo transport = content.getValue().transport;
1053 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1054
1055 // TODO check that credentials remained the same
1056
1057 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1058 final String sdp;
1059 try {
1060 sdp = candidate.toSdpAttribute(credentials.ufrag);
1061 } catch (final IllegalArgumentException e) {
1062 Log.d(
1063 Config.LOGTAG,
1064 id.account.getJid().asBareJid()
1065 + ": ignoring invalid ICE candidate "
1066 + e.getMessage());
1067 continue;
1068 }
1069 final int mLineIndex = indices.indexOf(sdpMid);
1070 if (mLineIndex < 0) {
1071 Log.w(
1072 Config.LOGTAG,
1073 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1074 }
1075 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1076 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1077 this.webRTCWrapper.addIceCandidate(iceCandidate);
1078 }
1079 }
1080
1081 private RtpContentMap getRemoteContentMap() {
1082 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1083 }
1084
1085 private RtpContentMap getLocalContentMap() {
1086 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1087 }
1088
1089 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1090 final Group originalGroup = rtpContentMap.group;
1091 final List<String> identificationTags =
1092 originalGroup == null
1093 ? rtpContentMap.getNames()
1094 : originalGroup.getIdentificationTags();
1095 if (identificationTags.size() == 0) {
1096 Log.w(
1097 Config.LOGTAG,
1098 id.account.getJid().asBareJid()
1099 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1100 }
1101 return identificationTags;
1102 }
1103
1104 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1105 final JinglePacket jinglePacket, final boolean expectVerification) {
1106 try {
1107 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1108 } catch (final Exception e) {
1109 return Futures.immediateFailedFuture(e);
1110 }
1111 }
1112
1113 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1114 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1115 Log.d(
1116 Config.LOGTAG,
1117 "receiveRtpContentMap("
1118 + receivedContentMap.getClass().getSimpleName()
1119 + ",expectVerification="
1120 + expectVerification
1121 + ")");
1122 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1123 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1124 id.account
1125 .getAxolotlService()
1126 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1127 return Futures.transform(
1128 future,
1129 omemoVerifiedPayload -> {
1130 // TODO test if an exception here triggers a correct abort
1131 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1132 Log.d(
1133 Config.LOGTAG,
1134 id.account.getJid().asBareJid()
1135 + ": received verifiable DTLS fingerprint via "
1136 + omemoVerification);
1137 return omemoVerifiedPayload.getPayload();
1138 },
1139 MoreExecutors.directExecutor());
1140 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1141 return Futures.immediateFailedFuture(
1142 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1143 } else {
1144 return Futures.immediateFuture(receivedContentMap);
1145 }
1146 }
1147
1148 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1149 if (isInitiator()) {
1150 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1151 return;
1152 }
1153 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1154 Futures.addCallback(
1155 future,
1156 new FutureCallback<>() {
1157 @Override
1158 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1159 receiveSessionInitiate(jinglePacket, rtpContentMap);
1160 }
1161
1162 @Override
1163 public void onFailure(@NonNull final Throwable throwable) {
1164 respondOk(jinglePacket);
1165 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1166 }
1167 },
1168 MoreExecutors.directExecutor());
1169 }
1170
1171 private void receiveSessionInitiate(
1172 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1173 try {
1174 contentMap.requireContentDescriptions();
1175 contentMap.requireDTLSFingerprint(true);
1176 } catch (final RuntimeException e) {
1177 Log.d(
1178 Config.LOGTAG,
1179 id.account.getJid().asBareJid() + ": improperly formatted contents",
1180 Throwables.getRootCause(e));
1181 respondOk(jinglePacket);
1182 sendSessionTerminate(Reason.of(e), e.getMessage());
1183 return;
1184 }
1185 Log.d(
1186 Config.LOGTAG,
1187 "processing session-init with " + contentMap.contents.size() + " contents");
1188 final State target;
1189 if (this.state == State.PROCEED) {
1190 Preconditions.checkState(
1191 proposedMedia != null && proposedMedia.size() > 0,
1192 "proposed media must be set when processing pre-approved session-initiate");
1193 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1194 sendSessionTerminate(
1195 Reason.SECURITY_ERROR,
1196 String.format(
1197 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1198 this.proposedMedia, contentMap.getMedia()));
1199 return;
1200 }
1201 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1202 } else {
1203 target = State.SESSION_INITIALIZED;
1204 setProposedMedia(contentMap.getMedia());
1205 }
1206 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1207 respondOk(jinglePacket);
1208 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1209 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1210 Log.d(
1211 Config.LOGTAG,
1212 id.account.getJid().asBareJid()
1213 + ": automatically accepting session-initiate");
1214 sendSessionAccept();
1215 } else {
1216 Log.d(
1217 Config.LOGTAG,
1218 id.account.getJid().asBareJid()
1219 + ": received not pre-approved session-initiate. start ringing");
1220 startRinging();
1221 }
1222 } else {
1223 Log.d(
1224 Config.LOGTAG,
1225 String.format(
1226 "%s: received session-initiate while in state %s",
1227 id.account.getJid().asBareJid(), state));
1228 terminateWithOutOfOrder(jinglePacket);
1229 }
1230 }
1231
1232 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1233 if (isResponder()) {
1234 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1235 return;
1236 }
1237 final ListenableFuture<RtpContentMap> future =
1238 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1239 Futures.addCallback(
1240 future,
1241 new FutureCallback<>() {
1242 @Override
1243 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1244 receiveSessionAccept(jinglePacket, rtpContentMap);
1245 }
1246
1247 @Override
1248 public void onFailure(@NonNull final Throwable throwable) {
1249 respondOk(jinglePacket);
1250 Log.d(
1251 Config.LOGTAG,
1252 id.account.getJid().asBareJid()
1253 + ": improperly formatted contents in session-accept",
1254 throwable);
1255 webRTCWrapper.close();
1256 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1257 }
1258 },
1259 MoreExecutors.directExecutor());
1260 }
1261
1262 private void receiveSessionAccept(
1263 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1264 try {
1265 contentMap.requireContentDescriptions();
1266 contentMap.requireDTLSFingerprint();
1267 } catch (final RuntimeException e) {
1268 respondOk(jinglePacket);
1269 Log.d(
1270 Config.LOGTAG,
1271 id.account.getJid().asBareJid()
1272 + ": improperly formatted contents in session-accept",
1273 e);
1274 webRTCWrapper.close();
1275 sendSessionTerminate(Reason.of(e), e.getMessage());
1276 return;
1277 }
1278 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1279 if (!initiatorMedia.equals(contentMap.getMedia())) {
1280 sendSessionTerminate(
1281 Reason.SECURITY_ERROR,
1282 String.format(
1283 "Your session-included included media %s but our session-initiate was %s",
1284 this.proposedMedia, contentMap.getMedia()));
1285 return;
1286 }
1287 Log.d(
1288 Config.LOGTAG,
1289 "processing session-accept with " + contentMap.contents.size() + " contents");
1290 if (transition(State.SESSION_ACCEPTED)) {
1291 respondOk(jinglePacket);
1292 receiveSessionAccept(contentMap);
1293 } else {
1294 Log.d(
1295 Config.LOGTAG,
1296 String.format(
1297 "%s: received session-accept while in state %s",
1298 id.account.getJid().asBareJid(), state));
1299 respondOk(jinglePacket);
1300 }
1301 }
1302
1303 private void receiveSessionAccept(final RtpContentMap contentMap) {
1304 this.responderRtpContentMap = contentMap;
1305 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1306 final SessionDescription sessionDescription;
1307 try {
1308 sessionDescription = SessionDescription.of(contentMap, false);
1309 } catch (final IllegalArgumentException | NullPointerException e) {
1310 Log.d(
1311 Config.LOGTAG,
1312 id.account.getJid().asBareJid()
1313 + ": unable convert offer from session-accept to SDP",
1314 e);
1315 webRTCWrapper.close();
1316 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1317 return;
1318 }
1319 final org.webrtc.SessionDescription answer =
1320 new org.webrtc.SessionDescription(
1321 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1322 try {
1323 this.webRTCWrapper.setRemoteDescription(answer).get();
1324 } catch (final Exception e) {
1325 Log.d(
1326 Config.LOGTAG,
1327 id.account.getJid().asBareJid()
1328 + ": unable to set remote description after receiving session-accept",
1329 Throwables.getRootCause(e));
1330 webRTCWrapper.close();
1331 sendSessionTerminate(
1332 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1333 return;
1334 }
1335 processCandidates(contentMap.contents.entrySet());
1336 }
1337
1338 private void sendSessionAccept() {
1339 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1340 if (rtpContentMap == null) {
1341 throw new IllegalStateException("initiator RTP Content Map has not been set");
1342 }
1343 final SessionDescription offer;
1344 try {
1345 offer = SessionDescription.of(rtpContentMap, true);
1346 } catch (final IllegalArgumentException | NullPointerException e) {
1347 Log.d(
1348 Config.LOGTAG,
1349 id.account.getJid().asBareJid()
1350 + ": unable convert offer from session-initiate to SDP",
1351 e);
1352 webRTCWrapper.close();
1353 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1354 return;
1355 }
1356 sendSessionAccept(rtpContentMap.getMedia(), offer);
1357 }
1358
1359 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1360 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1361 }
1362
1363 private synchronized void sendSessionAccept(
1364 final Set<Media> media,
1365 final SessionDescription offer,
1366 final List<PeerConnection.IceServer> iceServers) {
1367 if (isTerminated()) {
1368 Log.w(
1369 Config.LOGTAG,
1370 id.account.getJid().asBareJid()
1371 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1372 return;
1373 }
1374 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1375 try {
1376 setupWebRTC(media, iceServers, !includeCandidates);
1377 } catch (final WebRTCWrapper.InitializationException e) {
1378 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1379 webRTCWrapper.close();
1380 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1381 return;
1382 }
1383 final org.webrtc.SessionDescription sdp =
1384 new org.webrtc.SessionDescription(
1385 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1386 try {
1387 this.webRTCWrapper.setRemoteDescription(sdp).get();
1388 addIceCandidatesFromBlackLog();
1389 org.webrtc.SessionDescription webRTCSessionDescription =
1390 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1391 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1392 } catch (final Exception e) {
1393 failureToAcceptSession(e);
1394 }
1395 }
1396
1397 private void failureToAcceptSession(final Throwable throwable) {
1398 if (isTerminated()) {
1399 return;
1400 }
1401 final Throwable rootCause = Throwables.getRootCause(throwable);
1402 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1403 webRTCWrapper.close();
1404 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1405 }
1406
1407 private void failureToPerformAction(
1408 final JinglePacket.Action action, final Throwable throwable) {
1409 if (isTerminated()) {
1410 return;
1411 }
1412 final Throwable rootCause = Throwables.getRootCause(throwable);
1413 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1414 webRTCWrapper.close();
1415 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1416 }
1417
1418 private void addIceCandidatesFromBlackLog() {
1419 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1420 while ((foo = this.pendingIceCandidates.poll()) != null) {
1421 processCandidate(foo);
1422 Log.d(
1423 Config.LOGTAG,
1424 id.account.getJid().asBareJid() + ": added candidate from back log");
1425 }
1426 }
1427
1428 private void prepareSessionAccept(
1429 final org.webrtc.SessionDescription webRTCSessionDescription,
1430 final boolean includeCandidates) {
1431 final SessionDescription sessionDescription =
1432 SessionDescription.parse(webRTCSessionDescription.description);
1433 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1434 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1435 if (includeCandidates) {
1436 candidates = parseCandidates(sessionDescription);
1437 } else {
1438 candidates = ImmutableMultimap.of();
1439 }
1440 this.responderRtpContentMap = respondingRtpContentMap;
1441 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1442 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1443 prepareOutgoingContentMap(respondingRtpContentMap);
1444 Futures.addCallback(
1445 outgoingContentMapFuture,
1446 new FutureCallback<>() {
1447 @Override
1448 public void onSuccess(final RtpContentMap outgoingContentMap) {
1449 if (includeCandidates) {
1450 Log.d(
1451 Config.LOGTAG,
1452 "including "
1453 + candidates.size()
1454 + " candidates in session accept");
1455 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1456 } else {
1457 sendSessionAccept(outgoingContentMap);
1458 }
1459 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1460 }
1461
1462 @Override
1463 public void onFailure(@NonNull Throwable throwable) {
1464 failureToAcceptSession(throwable);
1465 }
1466 },
1467 MoreExecutors.directExecutor());
1468 }
1469
1470 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1471 if (isTerminated()) {
1472 Log.w(
1473 Config.LOGTAG,
1474 id.account.getJid().asBareJid()
1475 + ": preparing session accept was too slow. already terminated. nothing to do.");
1476 return;
1477 }
1478 transitionOrThrow(State.SESSION_ACCEPTED);
1479 final JinglePacket sessionAccept =
1480 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1481 send(sessionAccept);
1482 }
1483
1484 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1485 final RtpContentMap rtpContentMap) {
1486 if (this.omemoVerification.hasDeviceId()) {
1487 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1488 verifiedPayloadFuture =
1489 id.account
1490 .getAxolotlService()
1491 .encrypt(
1492 rtpContentMap,
1493 id.with,
1494 omemoVerification.getDeviceId());
1495 return Futures.transform(
1496 verifiedPayloadFuture,
1497 verifiedPayload -> {
1498 omemoVerification.setOrEnsureEqual(verifiedPayload);
1499 return verifiedPayload.getPayload();
1500 },
1501 MoreExecutors.directExecutor());
1502 } else {
1503 return Futures.immediateFuture(rtpContentMap);
1504 }
1505 }
1506
1507 synchronized void deliveryMessage(
1508 final Jid from,
1509 final Element message,
1510 final String serverMessageId,
1511 final long timestamp) {
1512 Log.d(
1513 Config.LOGTAG,
1514 id.account.getJid().asBareJid()
1515 + ": delivered message to JingleRtpConnection "
1516 + message);
1517 switch (message.getName()) {
1518 case "propose" -> receivePropose(
1519 from, Propose.upgrade(message), serverMessageId, timestamp);
1520 case "proceed" -> receiveProceed(
1521 from, Proceed.upgrade(message), serverMessageId, timestamp);
1522 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1523 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1524 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1525 }
1526 }
1527
1528 void deliverFailedProceed(final String message) {
1529 Log.d(
1530 Config.LOGTAG,
1531 id.account.getJid().asBareJid()
1532 + ": receive message error for proceed message ("
1533 + Strings.nullToEmpty(message)
1534 + ")");
1535 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1536 webRTCWrapper.close();
1537 Log.d(
1538 Config.LOGTAG,
1539 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1540 this.finish();
1541 }
1542 }
1543
1544 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1545 final boolean originatedFromMyself =
1546 from.asBareJid().equals(id.account.getJid().asBareJid());
1547 if (originatedFromMyself) {
1548 if (transition(State.ACCEPTED)) {
1549 acceptedOnOtherDevice(serverMsgId, timestamp);
1550 } else {
1551 Log.d(
1552 Config.LOGTAG,
1553 id.account.getJid().asBareJid()
1554 + ": unable to transition to accept because already in state="
1555 + this.state);
1556 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1557 }
1558 } else {
1559 Log.d(
1560 Config.LOGTAG,
1561 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1562 }
1563 }
1564
1565 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1566 if (serverMsgId != null) {
1567 this.message.setServerMsgId(serverMsgId);
1568 }
1569 this.message.setTime(timestamp);
1570 this.message.setCarbon(true); // indicate that call was accepted on other device
1571 this.writeLogMessageSuccess(0);
1572 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1573 this.finish();
1574 }
1575
1576 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1577 final boolean originatedFromMyself =
1578 from.asBareJid().equals(id.account.getJid().asBareJid());
1579 // reject from another one of my clients
1580 if (originatedFromMyself) {
1581 receiveRejectFromMyself(serverMsgId, timestamp);
1582 } else if (isInitiator()) {
1583 if (from.equals(id.with)) {
1584 receiveRejectFromResponder();
1585 } else {
1586 Log.d(
1587 Config.LOGTAG,
1588 id.account.getJid()
1589 + ": ignoring reject from "
1590 + from
1591 + " for session with "
1592 + id.with);
1593 }
1594 } else {
1595 Log.d(
1596 Config.LOGTAG,
1597 id.account.getJid()
1598 + ": ignoring reject from "
1599 + from
1600 + " for session with "
1601 + id.with);
1602 }
1603 }
1604
1605 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1606 if (transition(State.REJECTED)) {
1607 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1608 this.finish();
1609 if (serverMsgId != null) {
1610 this.message.setServerMsgId(serverMsgId);
1611 }
1612 this.message.setTime(timestamp);
1613 this.message.setCarbon(true); // indicate that call was rejected on other device
1614 writeLogMessageMissed();
1615 } else {
1616 Log.d(
1617 Config.LOGTAG,
1618 "not able to transition into REJECTED because already in " + this.state);
1619 }
1620 }
1621
1622 private void receiveRejectFromResponder() {
1623 if (isInState(State.PROCEED)) {
1624 Log.d(
1625 Config.LOGTAG,
1626 id.account.getJid()
1627 + ": received reject while still in proceed. callee reconsidered");
1628 closeTransitionLogFinish(State.REJECTED_RACED);
1629 return;
1630 }
1631 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1632 Log.d(
1633 Config.LOGTAG,
1634 id.account.getJid()
1635 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1636 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1637 return;
1638 }
1639 Log.d(
1640 Config.LOGTAG,
1641 id.account.getJid()
1642 + ": ignoring reject from responder because already in state "
1643 + this.state);
1644 }
1645
1646 private void receivePropose(
1647 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1648 final boolean originatedFromMyself =
1649 from.asBareJid().equals(id.account.getJid().asBareJid());
1650 if (originatedFromMyself) {
1651 Log.d(
1652 Config.LOGTAG,
1653 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1654 } else if (transition(
1655 State.PROPOSED,
1656 () -> {
1657 final Collection<RtpDescription> descriptions =
1658 Collections2.transform(
1659 Collections2.filter(
1660 propose.getDescriptions(),
1661 d -> d instanceof RtpDescription),
1662 input -> (RtpDescription) input);
1663 final Collection<Media> media =
1664 Collections2.transform(descriptions, RtpDescription::getMedia);
1665 Preconditions.checkState(
1666 !media.contains(Media.UNKNOWN),
1667 "RTP descriptions contain unknown media");
1668 Log.d(
1669 Config.LOGTAG,
1670 id.account.getJid().asBareJid()
1671 + ": received session proposal from "
1672 + from
1673 + " for "
1674 + media);
1675 this.setProposedMedia(Sets.newHashSet(media));
1676 })) {
1677 if (serverMsgId != null) {
1678 this.message.setServerMsgId(serverMsgId);
1679 }
1680 this.message.setTime(timestamp);
1681 startRinging();
1682 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1683 sendJingleMessage("ringing");
1684 }
1685 } else {
1686 Log.d(
1687 Config.LOGTAG,
1688 id.account.getJid()
1689 + ": ignoring session proposal because already in "
1690 + state);
1691 }
1692 }
1693
1694 private void startRinging() {
1695 this.callIntegration.setRinging();
1696 Log.d(
1697 Config.LOGTAG,
1698 id.account.getJid().asBareJid()
1699 + ": received call from "
1700 + id.with
1701 + ". start ringing");
1702 ringingTimeoutFuture =
1703 jingleConnectionManager.schedule(
1704 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1705 if (CallIntegration.selfManaged(xmppConnectionService)) {
1706 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1707 return;
1708 }
1709 }
1710 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1711 }
1712
1713 private synchronized void ringingTimeout() {
1714 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1715 switch (this.state) {
1716 case PROPOSED -> {
1717 message.markUnread();
1718 rejectCallFromProposed();
1719 }
1720 case SESSION_INITIALIZED -> {
1721 message.markUnread();
1722 rejectCallFromSessionInitiate();
1723 }
1724 }
1725 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1726 }
1727
1728 private void cancelRingingTimeout() {
1729 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1730 if (future != null && !future.isCancelled()) {
1731 future.cancel(false);
1732 }
1733 }
1734
1735 private void receiveProceed(
1736 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1737 final Set<Media> media =
1738 Preconditions.checkNotNull(
1739 this.proposedMedia, "Proposed media has to be set before handling proceed");
1740 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1741 if (from.equals(id.with)) {
1742 if (isInitiator()) {
1743 if (transition(State.PROCEED)) {
1744 if (serverMsgId != null) {
1745 this.message.setServerMsgId(serverMsgId);
1746 }
1747 this.message.setTime(timestamp);
1748 final Integer remoteDeviceId = proceed.getDeviceId();
1749 if (isOmemoEnabled()) {
1750 this.omemoVerification.setDeviceId(remoteDeviceId);
1751 } else {
1752 if (remoteDeviceId != null) {
1753 Log.d(
1754 Config.LOGTAG,
1755 id.account.getJid().asBareJid()
1756 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1757 }
1758 this.omemoVerification.setDeviceId(null);
1759 }
1760 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1761 } else {
1762 Log.d(
1763 Config.LOGTAG,
1764 String.format(
1765 "%s: ignoring proceed because already in %s",
1766 id.account.getJid().asBareJid(), this.state));
1767 }
1768 } else {
1769 Log.d(
1770 Config.LOGTAG,
1771 String.format(
1772 "%s: ignoring proceed because we were not initializing",
1773 id.account.getJid().asBareJid()));
1774 }
1775 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1776 if (transition(State.ACCEPTED)) {
1777 Log.d(
1778 Config.LOGTAG,
1779 id.account.getJid().asBareJid()
1780 + ": moved session with "
1781 + id.with
1782 + " into state accepted after received carbon copied proceed");
1783 acceptedOnOtherDevice(serverMsgId, timestamp);
1784 }
1785 } else {
1786 Log.d(
1787 Config.LOGTAG,
1788 String.format(
1789 "%s: ignoring proceed from %s. was expected from %s",
1790 id.account.getJid().asBareJid(), from, id.with));
1791 }
1792 }
1793
1794 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1795 if (from.equals(id.with)) {
1796 final State target =
1797 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1798 if (transition(target)) {
1799 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1800 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1801 Log.d(
1802 Config.LOGTAG,
1803 id.account.getJid().asBareJid()
1804 + ": session with "
1805 + id.with
1806 + " has been retracted (serverMsgId="
1807 + serverMsgId
1808 + ")");
1809 if (serverMsgId != null) {
1810 this.message.setServerMsgId(serverMsgId);
1811 }
1812 this.message.setTime(timestamp);
1813 if (target == State.RETRACTED) {
1814 this.message.markUnread();
1815 }
1816 writeLogMessageMissed();
1817 finish();
1818 } else {
1819 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1820 }
1821 } else {
1822 // TODO parse retract from self
1823 Log.d(
1824 Config.LOGTAG,
1825 id.account.getJid().asBareJid()
1826 + ": received retract from "
1827 + from
1828 + ". expected retract from"
1829 + id.with
1830 + ". ignoring");
1831 }
1832 }
1833
1834 public void sendSessionInitiate() {
1835 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1836 }
1837
1838 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1839 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1840 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1841 }
1842
1843 private synchronized void sendSessionInitiate(
1844 final Set<Media> media,
1845 final State targetState,
1846 final List<PeerConnection.IceServer> iceServers) {
1847 if (isTerminated()) {
1848 Log.w(
1849 Config.LOGTAG,
1850 id.account.getJid().asBareJid()
1851 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1852 return;
1853 }
1854 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1855 try {
1856 setupWebRTC(media, iceServers, !includeCandidates);
1857 } catch (final WebRTCWrapper.InitializationException e) {
1858 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1859 webRTCWrapper.close();
1860 sendRetract(Reason.ofThrowable(e));
1861 return;
1862 }
1863 try {
1864 org.webrtc.SessionDescription webRTCSessionDescription =
1865 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1866 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1867 } catch (final Exception e) {
1868 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1869 // exceptions
1870 failureToInitiateSession(e, targetState);
1871 }
1872 }
1873
1874 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1875 if (isTerminated()) {
1876 return;
1877 }
1878 Log.d(
1879 Config.LOGTAG,
1880 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1881 Throwables.getRootCause(throwable));
1882 webRTCWrapper.close();
1883 final Reason reason = Reason.ofThrowable(throwable);
1884 if (isInState(targetState)) {
1885 sendSessionTerminate(reason, throwable.getMessage());
1886 } else {
1887 sendRetract(reason);
1888 }
1889 }
1890
1891 private void sendRetract(final Reason reason) {
1892 // TODO embed reason into retract
1893 sendJingleMessage("retract", id.with.asBareJid());
1894 transitionOrThrow(reasonToState(reason));
1895 this.finish();
1896 }
1897
1898 private void prepareSessionInitiate(
1899 final org.webrtc.SessionDescription webRTCSessionDescription,
1900 final boolean includeCandidates,
1901 final State targetState) {
1902 final SessionDescription sessionDescription =
1903 SessionDescription.parse(webRTCSessionDescription.description);
1904 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1905 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1906 if (includeCandidates) {
1907 candidates = parseCandidates(sessionDescription);
1908 } else {
1909 candidates = ImmutableMultimap.of();
1910 }
1911 this.initiatorRtpContentMap = rtpContentMap;
1912 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1913 encryptSessionInitiate(rtpContentMap);
1914 Futures.addCallback(
1915 outgoingContentMapFuture,
1916 new FutureCallback<>() {
1917 @Override
1918 public void onSuccess(final RtpContentMap outgoingContentMap) {
1919 if (includeCandidates) {
1920 Log.d(
1921 Config.LOGTAG,
1922 "including "
1923 + candidates.size()
1924 + " candidates in session initiate");
1925 sendSessionInitiate(
1926 outgoingContentMap.withCandidates(candidates), targetState);
1927 } else {
1928 sendSessionInitiate(outgoingContentMap, targetState);
1929 }
1930 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1931 }
1932
1933 @Override
1934 public void onFailure(@NonNull final Throwable throwable) {
1935 failureToInitiateSession(throwable, targetState);
1936 }
1937 },
1938 MoreExecutors.directExecutor());
1939 }
1940
1941 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1942 if (isTerminated()) {
1943 Log.w(
1944 Config.LOGTAG,
1945 id.account.getJid().asBareJid()
1946 + ": preparing session was too slow. already terminated. nothing to do.");
1947 return;
1948 }
1949 this.transitionOrThrow(targetState);
1950 final JinglePacket sessionInitiate =
1951 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1952 send(sessionInitiate);
1953 }
1954
1955 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1956 final RtpContentMap rtpContentMap) {
1957 if (this.omemoVerification.hasDeviceId()) {
1958 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1959 verifiedPayloadFuture =
1960 id.account
1961 .getAxolotlService()
1962 .encrypt(
1963 rtpContentMap,
1964 id.with,
1965 omemoVerification.getDeviceId());
1966 final ListenableFuture<RtpContentMap> future =
1967 Futures.transform(
1968 verifiedPayloadFuture,
1969 verifiedPayload -> {
1970 omemoVerification.setSessionFingerprint(
1971 verifiedPayload.getFingerprint());
1972 return verifiedPayload.getPayload();
1973 },
1974 MoreExecutors.directExecutor());
1975 if (Config.REQUIRE_RTP_VERIFICATION) {
1976 return future;
1977 }
1978 return Futures.catching(
1979 future,
1980 CryptoFailedException.class,
1981 e -> {
1982 Log.w(
1983 Config.LOGTAG,
1984 id.account.getJid().asBareJid()
1985 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1986 e);
1987 return rtpContentMap;
1988 },
1989 MoreExecutors.directExecutor());
1990 } else {
1991 return Futures.immediateFuture(rtpContentMap);
1992 }
1993 }
1994
1995 protected void sendSessionTerminate(final Reason reason) {
1996 sendSessionTerminate(reason, null);
1997 }
1998
1999 protected void sendSessionTerminate(final Reason reason, final String text) {
2000 sendSessionTerminate(reason, text, this::writeLogMessage);
2001 sendJingleMessageFinish(reason);
2002 }
2003
2004 private void sendTransportInfo(
2005 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2006 final RtpContentMap transportInfo;
2007 try {
2008 final RtpContentMap rtpContentMap =
2009 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2010 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2011 } catch (final Exception e) {
2012 Log.d(
2013 Config.LOGTAG,
2014 id.account.getJid().asBareJid()
2015 + ": unable to prepare transport-info from candidate for content="
2016 + contentName);
2017 return;
2018 }
2019 final JinglePacket jinglePacket =
2020 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2021 send(jinglePacket);
2022 }
2023
2024 public RtpEndUserState getEndUserState() {
2025 switch (this.state) {
2026 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2027 if (isInitiator()) {
2028 return RtpEndUserState.RINGING;
2029 } else {
2030 return RtpEndUserState.INCOMING_CALL;
2031 }
2032 }
2033 case PROCEED -> {
2034 if (isInitiator()) {
2035 return RtpEndUserState.RINGING;
2036 } else {
2037 return RtpEndUserState.ACCEPTING_CALL;
2038 }
2039 }
2040 case SESSION_INITIALIZED_PRE_APPROVED -> {
2041 if (isInitiator()) {
2042 return RtpEndUserState.RINGING;
2043 } else {
2044 return RtpEndUserState.CONNECTING;
2045 }
2046 }
2047 case SESSION_ACCEPTED -> {
2048 final ContentAddition ca = getPendingContentAddition();
2049 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2050 return RtpEndUserState.INCOMING_CONTENT_ADD;
2051 }
2052 return getPeerConnectionStateAsEndUserState();
2053 }
2054 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2055 if (isInitiator()) {
2056 return RtpEndUserState.DECLINED_OR_BUSY;
2057 } else {
2058 return RtpEndUserState.ENDED;
2059 }
2060 }
2061 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2062 return RtpEndUserState.ENDED;
2063 }
2064 case RETRACTED_RACED -> {
2065 if (isInitiator()) {
2066 return RtpEndUserState.ENDED;
2067 } else {
2068 return RtpEndUserState.RETRACTED;
2069 }
2070 }
2071 case TERMINATED_CONNECTIVITY_ERROR -> {
2072 return zeroDuration()
2073 ? RtpEndUserState.CONNECTIVITY_ERROR
2074 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2075 }
2076 case TERMINATED_APPLICATION_FAILURE -> {
2077 return RtpEndUserState.APPLICATION_ERROR;
2078 }
2079 case TERMINATED_SECURITY_ERROR -> {
2080 return RtpEndUserState.SECURITY_ERROR;
2081 }
2082 }
2083 throw new IllegalStateException(
2084 String.format("%s has no equivalent EndUserState", this.state));
2085 }
2086
2087 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2088 final PeerConnection.PeerConnectionState state;
2089 try {
2090 state = webRTCWrapper.getState();
2091 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2092 // We usually close the WebRTCWrapper *before* transitioning so we might still
2093 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2094 return RtpEndUserState.ENDING_CALL;
2095 }
2096 return switch (state) {
2097 case CONNECTED -> RtpEndUserState.CONNECTED;
2098 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2099 case CLOSED -> RtpEndUserState.ENDING_CALL;
2100 default -> zeroDuration()
2101 ? RtpEndUserState.CONNECTIVITY_ERROR
2102 : RtpEndUserState.RECONNECTING;
2103 };
2104 }
2105
2106 private boolean isPeerConnectionConnected() {
2107 try {
2108 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2109 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2110 return false;
2111 }
2112 }
2113
2114 private void updateCallIntegrationState() {
2115 switch (this.state) {
2116 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2117 if (isInitiator()) {
2118 this.callIntegration.setDialing();
2119 } else {
2120 this.callIntegration.setRinging();
2121 }
2122 }
2123 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2124 if (isInitiator()) {
2125 this.callIntegration.setDialing();
2126 } else {
2127 this.callIntegration.setInitialized();
2128 }
2129 }
2130 case SESSION_ACCEPTED -> {
2131 if (isPeerConnectionConnected()) {
2132 this.callIntegration.setActive();
2133 } else {
2134 this.callIntegration.setInitialized();
2135 }
2136 }
2137 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2138 if (isInitiator()) {
2139 this.callIntegration.busy();
2140 } else {
2141 this.callIntegration.rejected();
2142 }
2143 }
2144 case TERMINATED_SUCCESS -> this.callIntegration.success();
2145 case ACCEPTED -> this.callIntegration.accepted();
2146 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2147 .retracted();
2148 case TERMINATED_CONNECTIVITY_ERROR,
2149 TERMINATED_APPLICATION_FAILURE,
2150 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2151 default -> throw new IllegalStateException(
2152 String.format("%s is not handled", this.state));
2153 }
2154 }
2155
2156 public ContentAddition getPendingContentAddition() {
2157 final RtpContentMap in = this.incomingContentAdd;
2158 final RtpContentMap out = this.outgoingContentAdd;
2159 if (out != null) {
2160 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2161 } else if (in != null) {
2162 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2163 } else {
2164 return null;
2165 }
2166 }
2167
2168 public Set<Media> getMedia() {
2169 final State current = getState();
2170 if (current == State.NULL) {
2171 if (isInitiator()) {
2172 return Preconditions.checkNotNull(
2173 this.proposedMedia, "RTP connection has not been initialized properly");
2174 }
2175 throw new IllegalStateException("RTP connection has not been initialized yet");
2176 }
2177 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2178 return Preconditions.checkNotNull(
2179 this.proposedMedia, "RTP connection has not been initialized properly");
2180 }
2181 final RtpContentMap localContentMap = getLocalContentMap();
2182 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2183 if (localContentMap != null) {
2184 return localContentMap.getMedia();
2185 } else if (initiatorContentMap != null) {
2186 return initiatorContentMap.getMedia();
2187 } else if (isTerminated()) {
2188 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2189 } else {
2190 return Preconditions.checkNotNull(
2191 this.proposedMedia, "RTP connection has not been initialized properly");
2192 }
2193 }
2194
2195 public boolean isVerified() {
2196 final String fingerprint = this.omemoVerification.getFingerprint();
2197 if (fingerprint == null) {
2198 return false;
2199 }
2200 final FingerprintStatus status =
2201 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2202 return status != null && status.isVerified();
2203 }
2204
2205 public boolean addMedia(final Media media) {
2206 final Set<Media> currentMedia = getMedia();
2207 if (currentMedia.contains(media)) {
2208 throw new IllegalStateException(String.format("%s has already been proposed", media));
2209 }
2210 // TODO add state protection - can only add while ACCEPTED or so
2211 Log.d(Config.LOGTAG, "adding media: " + media);
2212 return webRTCWrapper.addTrack(media);
2213 }
2214
2215 public synchronized void acceptCall() {
2216 switch (this.state) {
2217 case PROPOSED -> {
2218 cancelRingingTimeout();
2219 acceptCallFromProposed();
2220 }
2221 case SESSION_INITIALIZED -> {
2222 cancelRingingTimeout();
2223 acceptCallFromSessionInitialized();
2224 }
2225 case ACCEPTED -> Log.w(
2226 Config.LOGTAG,
2227 id.account.getJid().asBareJid()
2228 + ": the call has already been accepted with another client. UI was just lagging behind");
2229 case PROCEED, SESSION_ACCEPTED -> Log.w(
2230 Config.LOGTAG,
2231 id.account.getJid().asBareJid()
2232 + ": the call has already been accepted. user probably double tapped the UI");
2233 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2234 }
2235 }
2236
2237 public synchronized void rejectCall() {
2238 if (isTerminated()) {
2239 Log.w(
2240 Config.LOGTAG,
2241 id.account.getJid().asBareJid()
2242 + ": received rejectCall() when session has already been terminated. nothing to do");
2243 return;
2244 }
2245 switch (this.state) {
2246 case PROPOSED -> rejectCallFromProposed();
2247 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2248 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2249 }
2250 }
2251
2252 public synchronized void integrationFailure() {
2253 final var state = getState();
2254 if (state == State.PROPOSED) {
2255 Log.e(
2256 Config.LOGTAG,
2257 id.account.getJid().asBareJid()
2258 + ": failed call integration in state proposed");
2259 rejectCallFromProposed();
2260 } else if (state == State.SESSION_INITIALIZED) {
2261 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2262 this.webRTCWrapper.close();
2263 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2264 } else {
2265 throw new IllegalStateException(
2266 String.format("Can not fail integration in state %s", state));
2267 }
2268 }
2269
2270 public synchronized void endCall() {
2271 if (isTerminated()) {
2272 Log.w(
2273 Config.LOGTAG,
2274 id.account.getJid().asBareJid()
2275 + ": received endCall() when session has already been terminated. nothing to do");
2276 return;
2277 }
2278 if (isInState(State.PROPOSED) && isResponder()) {
2279 rejectCallFromProposed();
2280 return;
2281 }
2282 if (isInState(State.PROCEED)) {
2283 if (isInitiator()) {
2284 retractFromProceed();
2285 } else {
2286 rejectCallFromProceed();
2287 }
2288 return;
2289 }
2290 if (isInitiator()
2291 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2292 this.webRTCWrapper.close();
2293 sendSessionTerminate(Reason.CANCEL);
2294 return;
2295 }
2296 if (isInState(State.SESSION_INITIALIZED)) {
2297 rejectCallFromSessionInitiate();
2298 return;
2299 }
2300 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2301 this.webRTCWrapper.close();
2302 sendSessionTerminate(Reason.SUCCESS);
2303 return;
2304 }
2305 if (isInState(
2306 State.TERMINATED_APPLICATION_FAILURE,
2307 State.TERMINATED_CONNECTIVITY_ERROR,
2308 State.TERMINATED_DECLINED_OR_BUSY)) {
2309 Log.d(
2310 Config.LOGTAG,
2311 "ignoring request to end call because already in state " + this.state);
2312 return;
2313 }
2314 throw new IllegalStateException(
2315 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2316 }
2317
2318 private void retractFromProceed() {
2319 Log.d(Config.LOGTAG, "retract from proceed");
2320 this.sendJingleMessage("retract");
2321 closeTransitionLogFinish(State.RETRACTED_RACED);
2322 }
2323
2324 private void closeTransitionLogFinish(final State state) {
2325 this.webRTCWrapper.close();
2326 transitionOrThrow(state);
2327 writeLogMessage(state);
2328 finish();
2329 }
2330
2331 private void setupWebRTC(
2332 final Set<Media> media,
2333 final List<PeerConnection.IceServer> iceServers,
2334 final boolean trickle)
2335 throws WebRTCWrapper.InitializationException {
2336 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2337 this.webRTCWrapper.setup(this.xmppConnectionService);
2338 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2339 this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2340 }
2341
2342 private void acceptCallFromProposed() {
2343 transitionOrThrow(State.PROCEED);
2344 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2345 this.callIntegration.startAudioRouting();
2346 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2347 this.sendJingleMessage("proceed");
2348 }
2349
2350 private void rejectCallFromProposed() {
2351 transitionOrThrow(State.REJECTED);
2352 writeLogMessageMissed();
2353 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2354 this.sendJingleMessage("reject");
2355 finish();
2356 }
2357
2358 private void rejectCallFromProceed() {
2359 this.sendJingleMessage("reject");
2360 closeTransitionLogFinish(State.REJECTED_RACED);
2361 }
2362
2363 private void rejectCallFromSessionInitiate() {
2364 webRTCWrapper.close();
2365 sendSessionTerminate(Reason.DECLINE);
2366 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2367 }
2368
2369 private void sendJingleMessage(final String action) {
2370 sendJingleMessage(action, id.with);
2371 }
2372
2373 private void sendJingleMessage(final String action, final Jid to) {
2374 final MessagePacket messagePacket = new MessagePacket();
2375 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2376 messagePacket.setTo(to);
2377 final Element intent =
2378 messagePacket
2379 .addChild(action, Namespace.JINGLE_MESSAGE)
2380 .setAttribute("id", id.sessionId);
2381 if ("proceed".equals(action)) {
2382 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2383 if (isOmemoEnabled()) {
2384 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2385 final Element device =
2386 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2387 device.setAttribute("id", deviceId);
2388 }
2389 }
2390 messagePacket.addChild("store", "urn:xmpp:hints");
2391 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2392 }
2393
2394 private void sendJingleMessageFinish(final Reason reason) {
2395 final var account = id.getAccount();
2396 final MessagePacket messagePacket =
2397 xmppConnectionService
2398 .getMessageGenerator()
2399 .sessionFinish(id.with, id.sessionId, reason);
2400 xmppConnectionService.sendMessagePacket(account, messagePacket);
2401 }
2402
2403 private boolean isOmemoEnabled() {
2404 final Conversational conversational = message.getConversation();
2405 if (conversational instanceof Conversation) {
2406 return ((Conversation) conversational).getNextEncryption()
2407 == Message.ENCRYPTION_AXOLOTL;
2408 }
2409 return false;
2410 }
2411
2412 private void acceptCallFromSessionInitialized() {
2413 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2414 this.callIntegration.startAudioRouting();
2415 sendSessionAccept();
2416 }
2417
2418 @Override
2419 protected synchronized boolean transition(final State target, final Runnable runnable) {
2420 if (super.transition(target, runnable)) {
2421 updateEndUserState();
2422 updateOngoingCallNotification();
2423 return true;
2424 } else {
2425 return false;
2426 }
2427 }
2428
2429 @Override
2430 public void onIceCandidate(final IceCandidate iceCandidate) {
2431 final RtpContentMap rtpContentMap =
2432 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2433 final IceUdpTransportInfo.Credentials credentials;
2434 try {
2435 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2436 } catch (final IllegalArgumentException e) {
2437 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2438 return;
2439 }
2440 final String uFrag = credentials.ufrag;
2441 final IceUdpTransportInfo.Candidate candidate =
2442 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2443 if (candidate == null) {
2444 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2445 return;
2446 }
2447 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2448 sendTransportInfo(iceCandidate.sdpMid, candidate);
2449 }
2450
2451 @Override
2452 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2453 Log.d(
2454 Config.LOGTAG,
2455 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2456 this.stateHistory.add(newState);
2457 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2458 this.sessionDuration.start();
2459 updateOngoingCallNotification();
2460 } else if (this.sessionDuration.isRunning()) {
2461 this.sessionDuration.stop();
2462 updateOngoingCallNotification();
2463 }
2464
2465 final boolean neverConnected =
2466 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2467
2468 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2469 if (neverConnected) {
2470 if (isTerminated()) {
2471 Log.d(
2472 Config.LOGTAG,
2473 id.account.getJid().asBareJid()
2474 + ": not sending session-terminate after connectivity error because session is already in state "
2475 + this.state);
2476 return;
2477 }
2478 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2479 return;
2480 } else {
2481 this.restartIce();
2482 }
2483 }
2484 updateEndUserState();
2485 }
2486
2487 private void restartIce() {
2488 this.stateHistory.clear();
2489 this.webRTCWrapper.restartIceAsync();
2490 }
2491
2492 @Override
2493 public void onRenegotiationNeeded() {
2494 this.webRTCWrapper.execute(this::renegotiate);
2495 }
2496
2497 private synchronized void renegotiate() {
2498 final SessionDescription sessionDescription;
2499 try {
2500 sessionDescription = setLocalSessionDescription();
2501 } catch (final Exception e) {
2502 final Throwable cause = Throwables.getRootCause(e);
2503 webRTCWrapper.close();
2504 if (isTerminated()) {
2505 Log.d(
2506 Config.LOGTAG,
2507 "failed to renegotiate. session was already terminated",
2508 cause);
2509 return;
2510 }
2511 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2512 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2513 return;
2514 }
2515 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2516 final RtpContentMap currentContentMap = getLocalContentMap();
2517 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2518 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2519
2520 Log.d(
2521 Config.LOGTAG,
2522 id.getAccount().getJid().asBareJid()
2523 + ": renegotiate. iceRestart="
2524 + iceRestart
2525 + " content id diff="
2526 + diff);
2527
2528 if (diff.hasModifications() && iceRestart) {
2529 webRTCWrapper.close();
2530 sendSessionTerminate(
2531 Reason.FAILED_APPLICATION,
2532 "WebRTC unexpectedly tried to modify content and transport at once");
2533 return;
2534 }
2535
2536 if (iceRestart) {
2537 initiateIceRestart(rtpContentMap);
2538 return;
2539 } else if (diff.isEmpty()) {
2540 Log.d(
2541 Config.LOGTAG,
2542 "renegotiation. nothing to do. SignalingState="
2543 + this.webRTCWrapper.getSignalingState());
2544 }
2545
2546 if (diff.added.isEmpty()) {
2547 return;
2548 }
2549 modifyLocalContentMap(rtpContentMap);
2550 sendContentAdd(rtpContentMap, diff.added);
2551 }
2552
2553 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2554 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2555 final JinglePacket jinglePacket =
2556 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2557 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2558 jinglePacket.setTo(id.with);
2559 xmppConnectionService.sendIqPacket(
2560 id.account,
2561 jinglePacket,
2562 (account, response) -> {
2563 if (response.getType() == IqPacket.TYPE.RESULT) {
2564 Log.d(Config.LOGTAG, "received success to our ice restart");
2565 setLocalContentMap(rtpContentMap);
2566 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2567 return;
2568 }
2569 if (response.getType() == IqPacket.TYPE.ERROR) {
2570 if (isTieBreak(response)) {
2571 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2572 return;
2573 }
2574 handleIqErrorResponse(response);
2575 }
2576 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2577 handleIqTimeoutResponse(response);
2578 }
2579 });
2580 }
2581
2582 private boolean isTieBreak(final IqPacket response) {
2583 final Element error = response.findChild("error");
2584 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2585 }
2586
2587 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2588 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2589 this.outgoingContentAdd = contentAdd;
2590 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2591 prepareOutgoingContentMap(contentAdd);
2592 Futures.addCallback(
2593 outgoingContentMapFuture,
2594 new FutureCallback<>() {
2595 @Override
2596 public void onSuccess(final RtpContentMap outgoingContentMap) {
2597 sendContentAdd(outgoingContentMap);
2598 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2599 }
2600
2601 @Override
2602 public void onFailure(@NonNull Throwable throwable) {
2603 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2604 }
2605 },
2606 MoreExecutors.directExecutor());
2607 }
2608
2609 private void sendContentAdd(final RtpContentMap contentAdd) {
2610
2611 final JinglePacket jinglePacket =
2612 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2613 jinglePacket.setTo(id.with);
2614 xmppConnectionService.sendIqPacket(
2615 id.account,
2616 jinglePacket,
2617 (connection, response) -> {
2618 if (response.getType() == IqPacket.TYPE.RESULT) {
2619 Log.d(
2620 Config.LOGTAG,
2621 id.getAccount().getJid().asBareJid()
2622 + ": received ACK to our content-add");
2623 return;
2624 }
2625 if (response.getType() == IqPacket.TYPE.ERROR) {
2626 if (isTieBreak(response)) {
2627 this.outgoingContentAdd = null;
2628 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2629 return;
2630 }
2631 handleIqErrorResponse(response);
2632 }
2633 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2634 handleIqTimeoutResponse(response);
2635 }
2636 });
2637 }
2638
2639 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2640 if (isInitiator()) {
2641 this.initiatorRtpContentMap = rtpContentMap;
2642 } else {
2643 this.responderRtpContentMap = rtpContentMap;
2644 }
2645 }
2646
2647 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2648 if (isInitiator()) {
2649 this.responderRtpContentMap = rtpContentMap;
2650 } else {
2651 this.initiatorRtpContentMap = rtpContentMap;
2652 }
2653 }
2654
2655 // this method is to be used for content map modifications that modify media
2656 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2657 final RtpContentMap activeContents = rtpContentMap.activeContents();
2658 setLocalContentMap(activeContents);
2659 this.callIntegration.setAudioDeviceWhenAvailable(
2660 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2661 updateEndUserState();
2662 }
2663
2664 private SessionDescription setLocalSessionDescription()
2665 throws ExecutionException, InterruptedException {
2666 final org.webrtc.SessionDescription sessionDescription =
2667 this.webRTCWrapper.setLocalDescription(false).get();
2668 return SessionDescription.parse(sessionDescription.description);
2669 }
2670
2671 private void closeWebRTCSessionAfterFailedConnection() {
2672 this.webRTCWrapper.close();
2673 synchronized (this) {
2674 if (isTerminated()) {
2675 Log.d(
2676 Config.LOGTAG,
2677 id.account.getJid().asBareJid()
2678 + ": no need to send session-terminate after failed connection. Other party already did");
2679 return;
2680 }
2681 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2682 }
2683 }
2684
2685 public boolean zeroDuration() {
2686 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2687 }
2688
2689 public long getCallDuration() {
2690 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2691 }
2692
2693 @Override
2694 public CallIntegration getCallIntegration() {
2695 return this.callIntegration;
2696 }
2697
2698 public boolean isMicrophoneEnabled() {
2699 return webRTCWrapper.isMicrophoneEnabled();
2700 }
2701
2702 public boolean setMicrophoneEnabled(final boolean enabled) {
2703 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2704 }
2705
2706 public boolean isVideoEnabled() {
2707 return webRTCWrapper.isVideoEnabled();
2708 }
2709
2710 public void setVideoEnabled(final boolean enabled) {
2711 webRTCWrapper.setVideoEnabled(enabled);
2712 }
2713
2714 public boolean isCameraSwitchable() {
2715 return webRTCWrapper.isCameraSwitchable();
2716 }
2717
2718 public boolean isFrontCamera() {
2719 return webRTCWrapper.isFrontCamera();
2720 }
2721
2722 public ListenableFuture<Boolean> switchCamera() {
2723 return webRTCWrapper.switchCamera();
2724 }
2725
2726 @Override
2727 public synchronized void onCallIntegrationShowIncomingCallUi() {
2728 if (isTerminated()) {
2729 // there might be race conditions with the call integration service invoking this
2730 // callback when the rtp session has already ended.
2731 Log.w(
2732 Config.LOGTAG,
2733 "CallIntegration requested incoming call UI but session was already terminated");
2734 return;
2735 }
2736 // TODO apparently this can be called too early as well?
2737 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2738 }
2739
2740 @Override
2741 public void onCallIntegrationDisconnect() {
2742 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2743 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2744 rejectCall();
2745 } else {
2746 endCall();
2747 }
2748 }
2749
2750 @Override
2751 public void onCallIntegrationReject() {
2752 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2753 try {
2754 rejectCall();
2755 } catch (final IllegalStateException e) {
2756 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2757 }
2758 }
2759
2760 @Override
2761 public void onCallIntegrationAnswer() {
2762 // we need to start the UI to a) show it and b) be able to ask for permissions
2763 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2764 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2765 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2766 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2767 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2768 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2769 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2770 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2771 xmppConnectionService.startActivity(intent);
2772 }
2773
2774 @Override
2775 public void onCallIntegrationSilence() {
2776 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2777 }
2778
2779 @Override
2780 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2781 this.webRTCWrapper.setMicrophoneEnabled(enabled);
2782 }
2783
2784 @Override
2785 public void onAudioDeviceChanged(
2786 final CallIntegration.AudioDevice selectedAudioDevice,
2787 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2788 Log.d(
2789 Config.LOGTAG,
2790 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2791 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2792 selectedAudioDevice, availableAudioDevices);
2793 }
2794
2795 private void updateEndUserState() {
2796 final RtpEndUserState endUserState = getEndUserState();
2797 this.updateCallIntegrationState();
2798 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2799 id.account, id.with, id.sessionId, endUserState);
2800 }
2801
2802 private void updateOngoingCallNotification() {
2803 final State state = this.state;
2804 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2805 final boolean reconnecting;
2806 if (state == State.SESSION_ACCEPTED) {
2807 reconnecting =
2808 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2809 } else {
2810 reconnecting = false;
2811 }
2812 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2813 } else {
2814 xmppConnectionService.removeOngoingCall();
2815 }
2816 }
2817
2818 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2819 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2820 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2821 request.setTo(id.account.getDomain());
2822 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2823 xmppConnectionService.sendIqPacket(
2824 id.account,
2825 request,
2826 (account, response) -> {
2827 final var iceServers = IceServers.parse(response);
2828 if (iceServers.isEmpty()) {
2829 Log.w(
2830 Config.LOGTAG,
2831 id.account.getJid().asBareJid()
2832 + ": no ICE server found "
2833 + response);
2834 }
2835 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2836 });
2837 } else {
2838 Log.w(
2839 Config.LOGTAG,
2840 id.account.getJid().asBareJid() + ": has no external service discovery");
2841 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2842 }
2843 }
2844
2845 @Override
2846 protected void terminateTransport() {
2847 this.webRTCWrapper.close();
2848 }
2849
2850 @Override
2851 protected void finish() {
2852 if (isTerminated()) {
2853 this.cancelRingingTimeout();
2854 this.callIntegration.verifyDisconnected();
2855 this.webRTCWrapper.verifyClosed();
2856 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2857 super.finish();
2858 try {
2859 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2860 log.getParentFile().mkdirs();
2861 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2862 } catch (final IOException e) { }
2863 } else {
2864 throw new IllegalStateException(
2865 String.format("Unable to call finish from %s", this.state));
2866 }
2867 }
2868
2869 private void writeLogMessage(final State state) {
2870 final long duration = getCallDuration();
2871 if (state == State.TERMINATED_SUCCESS
2872 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2873 writeLogMessageSuccess(duration);
2874 } else {
2875 writeLogMessageMissed();
2876 }
2877 }
2878
2879 private void writeLogMessageSuccess(final long duration) {
2880 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2881 this.writeMessage();
2882 }
2883
2884 private void writeLogMessageMissed() {
2885 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2886 this.writeMessage();
2887 }
2888
2889 private void writeMessage() {
2890 final Conversational conversational = message.getConversation();
2891 if (conversational instanceof Conversation) {
2892 ((Conversation) conversational).add(this.message);
2893 xmppConnectionService.createMessageAsync(message);
2894 xmppConnectionService.updateConversationUi();
2895 } else {
2896 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2897 }
2898 }
2899
2900 public Optional<VideoTrack> getLocalVideoTrack() {
2901 return webRTCWrapper.getLocalVideoTrack();
2902 }
2903
2904 public Optional<VideoTrack> getRemoteVideoTrack() {
2905 return webRTCWrapper.getRemoteVideoTrack();
2906 }
2907
2908 public EglBase.Context getEglBaseContext() {
2909 return webRTCWrapper.getEglBaseContext();
2910 }
2911
2912 void setProposedMedia(final Set<Media> media) {
2913 this.proposedMedia = media;
2914 this.callIntegration.setVideoState(
2915 Media.audioOnly(media)
2916 ? VideoProfile.STATE_AUDIO_ONLY
2917 : VideoProfile.STATE_BIDIRECTIONAL);
2918 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2919 }
2920
2921 public void fireStateUpdate() {
2922 final RtpEndUserState endUserState = getEndUserState();
2923 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2924 id.account, id.with, id.sessionId, endUserState);
2925 }
2926
2927 public boolean isSwitchToVideoAvailable() {
2928 final boolean prerequisite =
2929 Media.audioOnly(getMedia())
2930 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2931 .contains(getEndUserState());
2932 return prerequisite && remoteHasVideoFeature();
2933 }
2934
2935 private boolean remoteHasVideoFeature() {
2936 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2937 }
2938
2939 private boolean remoteHasSdpOfferAnswer() {
2940 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2941 }
2942
2943 @Override
2944 public Account getAccount() {
2945 return id.account;
2946 }
2947
2948 @Override
2949 public Jid getWith() {
2950 return id.with;
2951 }
2952
2953 @Override
2954 public String getSessionId() {
2955 return id.sessionId;
2956 }
2957
2958 private interface OnIceServersDiscovered {
2959 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2960 }
2961}